diff --git a/bin/glitch_patterns.py b/bin/glitch_patterns.py new file mode 100644 index 00000000..42dc3c9d --- /dev/null +++ b/bin/glitch_patterns.py @@ -0,0 +1,297 @@ +""" +Glitch pattern definitions for 3D world anomaly detection. + +Defines known visual artifact categories commonly found in 3D web worlds, +particularly The Matrix environments. Each pattern includes detection +heuristics and severity ratings. +""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +class GlitchSeverity(Enum): + CRITICAL = "critical" + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + INFO = "info" + + +class GlitchCategory(Enum): + FLOATING_ASSETS = "floating_assets" + Z_FIGHTING = "z_fighting" + MISSING_TEXTURES = "missing_textures" + CLIPPING = "clipping" + BROKEN_NORMALS = "broken_normals" + SHADOW_ARTIFACTS = "shadow_artifacts" + LIGHTMAP_ERRORS = "lightmap_errors" + LOD_POPPING = "lod_popping" + WATER_REFLECTION = "water_reflection" + SKYBOX_SEAM = "skybox_seam" + + +@dataclass +class GlitchPattern: + """Definition of a known glitch pattern with detection parameters.""" + category: GlitchCategory + name: str + description: str + severity: GlitchSeverity + detection_prompts: list[str] + visual_indicators: list[str] + confidence_threshold: float = 0.6 + + def to_dict(self) -> dict: + return { + "category": self.category.value, + "name": self.name, + "description": self.description, + "severity": self.severity.value, + "detection_prompts": self.detection_prompts, + "visual_indicators": self.visual_indicators, + "confidence_threshold": self.confidence_threshold, + } + + +# Known glitch patterns for Matrix 3D world scanning +MATRIX_GLITCH_PATTERNS: list[GlitchPattern] = [ + GlitchPattern( + category=GlitchCategory.FLOATING_ASSETS, + name="Floating Object", + description="Object not properly grounded or anchored to the scene geometry. " + "Common in procedurally placed assets or after physics desync.", + severity=GlitchSeverity.HIGH, + detection_prompts=[ + "Identify any objects that appear to float above the ground without support.", + "Look for furniture, props, or geometry suspended in mid-air with no visible attachment.", + "Check for objects whose shadows do not align with the surface below them.", + ], + visual_indicators=[ + "gap between object base and surface", + "shadow detached from object", + "object hovering with no structural support", + ], + confidence_threshold=0.65, + ), + GlitchPattern( + category=GlitchCategory.Z_FIGHTING, + name="Z-Fighting Flicker", + description="Two coplanar surfaces competing for depth priority, causing " + "visible flickering or shimmering textures.", + severity=GlitchSeverity.MEDIUM, + detection_prompts=[ + "Look for surfaces that appear to shimmer, flicker, or show mixed textures.", + "Identify areas where two textures seem to overlap and compete for visibility.", + "Check walls, floors, or objects for surface noise or pattern interference.", + ], + visual_indicators=[ + "shimmering surface", + "texture flicker between two patterns", + "noisy flat surfaces", + "moire-like patterns on planar geometry", + ], + confidence_threshold=0.55, + ), + GlitchPattern( + category=GlitchCategory.MISSING_TEXTURES, + name="Missing or Placeholder Texture", + description="A surface rendered with a fallback checkerboard, solid magenta, " + "or the default engine placeholder texture.", + severity=GlitchSeverity.CRITICAL, + detection_prompts=[ + "Look for bright magenta, checkerboard, or solid-color surfaces that look out of place.", + "Identify any surfaces that appear as flat untextured colors inconsistent with the scene.", + "Check for black, white, or magenta patches where detailed textures should be.", + ], + visual_indicators=[ + "magenta/pink solid color surface", + "checkerboard pattern", + "flat single-color geometry", + "UV-debug texture visible", + ], + confidence_threshold=0.7, + ), + GlitchPattern( + category=GlitchCategory.CLIPPING, + name="Geometry Clipping", + description="Objects passing through each other or intersecting in physically " + "impossible ways due to collision mesh errors.", + severity=GlitchSeverity.HIGH, + detection_prompts=[ + "Look for objects that visibly pass through other objects (walls, floors, furniture).", + "Identify characters or props embedded inside geometry where they should not be.", + "Check for intersecting meshes where solid objects overlap unnaturally.", + ], + visual_indicators=[ + "object passing through wall or floor", + "embedded geometry", + "overlapping solid meshes", + "character limb inside furniture", + ], + confidence_threshold=0.6, + ), + GlitchPattern( + category=GlitchCategory.BROKEN_NORMALS, + name="Broken Surface Normals", + description="Inverted or incorrect surface normals causing faces to appear " + "inside-out, invisible from certain angles, or lit incorrectly.", + severity=GlitchSeverity.MEDIUM, + detection_prompts=[ + "Look for surfaces that appear dark or black on one side while lit on the other.", + "Identify objects that seem to vanish when viewed from certain angles.", + "Check for inverted shading where lit areas should be in shadow.", + ], + visual_indicators=[ + "dark/unlit face on otherwise lit model", + "invisible surface from one direction", + "inverted shadow gradient", + "inside-out appearance", + ], + confidence_threshold=0.5, + ), + GlitchPattern( + category=GlitchCategory.SHADOW_ARTIFACTS, + name="Shadow Artifact", + description="Broken, detached, or incorrectly rendered shadows that do not " + "match the casting geometry or scene lighting.", + severity=GlitchSeverity.LOW, + detection_prompts=[ + "Look for shadows that do not match the shape of nearby objects.", + "Identify shadow acne: banding or striped patterns on surfaces.", + "Check for floating shadows detached from any visible caster.", + ], + visual_indicators=[ + "shadow shape mismatch", + "shadow acne bands", + "detached floating shadow", + "Peter Panning (shadow offset from base)", + ], + confidence_threshold=0.5, + ), + GlitchPattern( + category=GlitchCategory.LOD_POPPING, + name="LOD Transition Pop", + description="Visible pop-in when level-of-detail models switch abruptly, " + "causing geometry or textures to change suddenly.", + severity=GlitchSeverity.LOW, + detection_prompts=[ + "Look for areas where mesh detail changes abruptly at visible boundaries.", + "Identify objects that appear to morph or shift geometry suddenly.", + "Check for texture resolution changes that create visible seams.", + ], + visual_indicators=[ + "visible mesh simplification boundary", + "texture resolution jump", + "geometry pop-in artifacts", + ], + confidence_threshold=0.45, + ), + GlitchPattern( + category=GlitchCategory.LIGHTMAP_ERRORS, + name="Lightmap Baking Error", + description="Incorrect or missing baked lighting causing dark spots, light " + "leaks, or mismatched illumination on static geometry.", + severity=GlitchSeverity.MEDIUM, + detection_prompts=[ + "Look for unusually dark patches on walls or ceilings that should be lit.", + "Identify bright light leaks through solid geometry seams.", + "Check for mismatched lighting between adjacent surfaces.", + ], + visual_indicators=[ + "dark splotch on lit surface", + "bright line at geometry seam", + "lighting discontinuity between adjacent faces", + ], + confidence_threshold=0.5, + ), + GlitchPattern( + category=GlitchCategory.WATER_REFLECTION, + name="Water/Reflection Error", + description="Incorrect reflections, missing water surfaces, or broken " + "reflection probe assignments.", + severity=GlitchSeverity.MEDIUM, + detection_prompts=[ + "Look for reflections that do not match the surrounding environment.", + "Identify water surfaces that appear solid or incorrectly rendered.", + "Check for mirror surfaces showing wrong scene geometry.", + ], + visual_indicators=[ + "reflection mismatch", + "solid water surface", + "incorrect environment map", + ], + confidence_threshold=0.5, + ), + GlitchPattern( + category=GlitchCategory.SKYBOX_SEAM, + name="Skybox Seam", + description="Visible seams or color mismatches at the edges of skybox cubemap faces.", + severity=GlitchSeverity.LOW, + detection_prompts=[ + "Look at the edges of the sky for visible seams or color shifts.", + "Identify discontinuities where skybox faces meet.", + "Check for texture stretching at skybox corners.", + ], + visual_indicators=[ + "visible line in sky", + "color discontinuity at sky edge", + "sky texture seam", + ], + confidence_threshold=0.45, + ), +] + + +def get_patterns_by_severity(min_severity: GlitchSeverity) -> list[GlitchPattern]: + """Return patterns at or above the given severity level.""" + severity_order = [ + GlitchSeverity.INFO, + GlitchSeverity.LOW, + GlitchSeverity.MEDIUM, + GlitchSeverity.HIGH, + GlitchSeverity.CRITICAL, + ] + min_idx = severity_order.index(min_severity) + return [p for p in MATRIX_GLITCH_PATTERNS if severity_order.index(p.severity) >= min_idx] + + +def get_pattern_by_category(category: GlitchCategory) -> Optional[GlitchPattern]: + """Return the pattern definition for a specific category.""" + for p in MATRIX_GLITCH_PATTERNS: + if p.category == category: + return p + return None + + +def build_vision_prompt(patterns: list[GlitchPattern] | None = None) -> str: + """Build a composite vision analysis prompt from pattern definitions.""" + if patterns is None: + patterns = MATRIX_GLITCH_PATTERNS + + sections = [] + for p in patterns: + prompt_text = " ".join(p.detection_prompts) + indicators = ", ".join(p.visual_indicators) + sections.append( + f"[{p.category.value.upper()}] {p.name} (severity: {p.severity.value})\n" + f" {p.description}\n" + f" Look for: {prompt_text}\n" + f" Visual indicators: {indicators}" + ) + + return ( + "Analyze this 3D world screenshot for visual glitches and artifacts. " + "For each detected issue, report the category, description of what you see, " + "approximate location in the image (x%, y%), and confidence (0.0-1.0).\n\n" + "Known glitch patterns to check:\n\n" + "\n\n".join(sections) + ) + + +if __name__ == "__main__": + import json + print(f"Loaded {len(MATRIX_GLITCH_PATTERNS)} glitch patterns:\n") + for p in MATRIX_GLITCH_PATTERNS: + print(f" [{p.severity.value:8s}] {p.category.value}: {p.name}") + print(f"\nVision prompt preview:\n{build_vision_prompt()[:500]}...") diff --git a/bin/matrix_glitch_detector.py b/bin/matrix_glitch_detector.py new file mode 100644 index 00000000..03f16e55 --- /dev/null +++ b/bin/matrix_glitch_detector.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python3 +""" +Matrix 3D World Glitch Detector + +Scans a 3D web world for visual artifacts using browser automation +and vision AI analysis. Produces structured glitch reports. + +Usage: + python matrix_glitch_detector.py [--angles 4] [--output report.json] + python matrix_glitch_detector.py --demo # Run with synthetic test data + +Ref: timmy-config#491 +""" + +import argparse +import base64 +import json +import os +import sys +import time +import uuid +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +# Add parent for glitch_patterns import +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from glitch_patterns import ( + GlitchCategory, + GlitchPattern, + GlitchSeverity, + MATRIX_GLITCH_PATTERNS, + build_vision_prompt, + get_patterns_by_severity, +) + + +@dataclass +class DetectedGlitch: + """A single detected glitch with metadata.""" + id: str + category: str + name: str + description: str + severity: str + confidence: float + location_x: Optional[float] = None # percentage across image + location_y: Optional[float] = None # percentage down image + screenshot_index: int = 0 + screenshot_angle: str = "front" + timestamp: str = "" + + def __post_init__(self): + if not self.timestamp: + self.timestamp = datetime.now(timezone.utc).isoformat() + + +@dataclass +class ScanResult: + """Complete scan result for a 3D world URL.""" + scan_id: str + url: str + timestamp: str + total_screenshots: int + angles_captured: list[str] + glitches: list[dict] = field(default_factory=list) + summary: dict = field(default_factory=dict) + metadata: dict = field(default_factory=dict) + + def to_json(self, indent: int = 2) -> str: + return json.dumps(asdict(self), indent=indent) + + +def generate_scan_angles(num_angles: int) -> list[dict]: + """Generate camera angle configurations for multi-angle scanning. + + Returns a list of dicts with yaw/pitch/label for browser camera control. + """ + base_angles = [ + {"yaw": 0, "pitch": 0, "label": "front"}, + {"yaw": 90, "pitch": 0, "label": "right"}, + {"yaw": 180, "pitch": 0, "label": "back"}, + {"yaw": 270, "pitch": 0, "label": "left"}, + {"yaw": 0, "pitch": -30, "label": "front_low"}, + {"yaw": 45, "pitch": -15, "label": "front_right_low"}, + {"yaw": 0, "pitch": 30, "label": "front_high"}, + {"yaw": 45, "pitch": 0, "label": "front_right"}, + ] + + if num_angles <= len(base_angles): + return base_angles[:num_angles] + return base_angles + [ + {"yaw": i * (360 // num_angles), "pitch": 0, "label": f"angle_{i}"} + for i in range(len(base_angles), num_angles) + ] + + +def capture_screenshots(url: str, angles: list[dict], output_dir: Path) -> list[Path]: + """Capture screenshots of a 3D web world from multiple angles. + + Uses browser_vision tool when available; falls back to placeholder generation + for testing and environments without browser access. + """ + output_dir.mkdir(parents=True, exist_ok=True) + screenshots = [] + + for i, angle in enumerate(angles): + filename = output_dir / f"screenshot_{i:03d}_{angle['label']}.png" + + # Attempt browser-based capture via browser_vision + try: + result = _browser_capture(url, angle, filename) + if result: + screenshots.append(filename) + continue + except Exception: + pass + + # Generate placeholder screenshot for offline/test scenarios + _generate_placeholder_screenshot(filename, angle) + screenshots.append(filename) + + return screenshots + + +def _browser_capture(url: str, angle: dict, output_path: Path) -> bool: + """Capture a screenshot via browser automation. + + This is a stub that delegates to the browser_vision tool when run + in an environment that provides it. In CI or offline mode, returns False. + """ + # Check if browser_vision is available via environment + bv_script = os.environ.get("BROWSER_VISION_SCRIPT") + if bv_script and Path(bv_script).exists(): + import subprocess + cmd = [ + sys.executable, bv_script, + "--url", url, + "--screenshot", str(output_path), + "--rotate-yaw", str(angle["yaw"]), + "--rotate-pitch", str(angle["pitch"]), + ] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + return proc.returncode == 0 and output_path.exists() + return False + + +def _generate_placeholder_screenshot(path: Path, angle: dict): + """Generate a minimal 1x1 PNG as a placeholder for testing.""" + # Minimal valid PNG (1x1 transparent pixel) + png_data = ( + b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01" + b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89" + b"\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01" + b"\r\n\xb4\x00\x00\x00\x00IEND\xaeB`\x82" + ) + path.write_bytes(png_data) + + +def analyze_with_vision( + screenshot_paths: list[Path], + angles: list[dict], + patterns: list[GlitchPattern] | None = None, +) -> list[DetectedGlitch]: + """Send screenshots to vision AI for glitch analysis. + + In environments with a vision model available, sends each screenshot + with the composite detection prompt. Otherwise returns simulated results. + """ + if patterns is None: + patterns = MATRIX_GLITCH_PATTERNS + + prompt = build_vision_prompt(patterns) + glitches = [] + + for i, (path, angle) in enumerate(zip(screenshot_paths, angles)): + # Attempt vision analysis + detected = _vision_analyze_image(path, prompt, i, angle["label"]) + glitches.extend(detected) + + return glitches + + +def _vision_analyze_image( + image_path: Path, + prompt: str, + screenshot_index: int, + angle_label: str, +) -> list[DetectedGlitch]: + """Analyze a single screenshot with vision AI. + + Uses the vision_analyze tool when available; returns empty list otherwise. + """ + # Check for vision API configuration + api_key = os.environ.get("VISION_API_KEY") or os.environ.get("OPENAI_API_KEY") + api_base = os.environ.get("VISION_API_BASE", "https://api.openai.com/v1") + + if api_key: + try: + return _call_vision_api( + image_path, prompt, screenshot_index, angle_label, api_key, api_base + ) + except Exception as e: + print(f" [!] Vision API error for {image_path.name}: {e}", file=sys.stderr) + + # No vision backend available + return [] + + +def _call_vision_api( + image_path: Path, + prompt: str, + screenshot_index: int, + angle_label: str, + api_key: str, + api_base: str, +) -> list[DetectedGlitch]: + """Call a vision API (OpenAI-compatible) for image analysis.""" + import urllib.request + import urllib.error + + image_data = base64.b64encode(image_path.read_bytes()).decode() + + payload = json.dumps({ + "model": os.environ.get("VISION_MODEL", "gpt-4o"), + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{image_data}", + "detail": "high", + }, + }, + ], + } + ], + "max_tokens": 4096, + }).encode() + + req = urllib.request.Request( + f"{api_base}/chat/completions", + data=payload, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + ) + + with urllib.request.urlopen(req, timeout=60) as resp: + result = json.loads(resp.read()) + + content = result["choices"][0]["message"]["content"] + return _parse_vision_response(content, screenshot_index, angle_label) + + +def _add_glitch_from_dict( + item: dict, + glitches: list[DetectedGlitch], + screenshot_index: int, + angle_label: str, +): + """Convert a dict from vision API response into a DetectedGlitch.""" + cat = item.get("category", item.get("type", "unknown")) + conf = float(item.get("confidence", item.get("score", 0.5))) + + glitch = DetectedGlitch( + id=str(uuid.uuid4())[:8], + category=cat, + name=item.get("name", item.get("label", cat)), + description=item.get("description", item.get("detail", "")), + severity=item.get("severity", _infer_severity(cat, conf)), + confidence=conf, + location_x=item.get("location_x", item.get("x")), + location_y=item.get("location_y", item.get("y")), + screenshot_index=screenshot_index, + screenshot_angle=angle_label, + ) + glitches.append(glitch) + + +def _parse_vision_response( + text: str, screenshot_index: int, angle_label: str +) -> list[DetectedGlitch]: + """Parse vision AI response into structured glitch detections.""" + glitches = [] + + # Try to extract JSON from the response + json_blocks = [] + in_json = False + json_buf = [] + + for line in text.split("\n"): + stripped = line.strip() + if stripped.startswith("```"): + if in_json and json_buf: + try: + json_blocks.append(json.loads("\n".join(json_buf))) + except json.JSONDecodeError: + pass + json_buf = [] + in_json = not in_json + continue + if in_json: + json_buf.append(line) + + # Flush any remaining buffer + if in_json and json_buf: + try: + json_blocks.append(json.loads("\n".join(json_buf))) + except json.JSONDecodeError: + pass + + # Also try parsing the entire response as JSON + try: + parsed = json.loads(text) + if isinstance(parsed, list): + json_blocks.extend(parsed) + elif isinstance(parsed, dict): + if "glitches" in parsed: + json_blocks.extend(parsed["glitches"]) + elif "detections" in parsed: + json_blocks.extend(parsed["detections"]) + else: + json_blocks.append(parsed) + except json.JSONDecodeError: + pass + + for item in json_blocks: + # Flatten arrays of detections + if isinstance(item, list): + for sub in item: + if isinstance(sub, dict): + _add_glitch_from_dict(sub, glitches, screenshot_index, angle_label) + elif isinstance(item, dict): + _add_glitch_from_dict(item, glitches, screenshot_index, angle_label) + + return glitches + + +def _infer_severity(category: str, confidence: float) -> str: + """Infer severity from category and confidence when not provided.""" + critical_cats = {"missing_textures", "clipping"} + high_cats = {"floating_assets", "broken_normals"} + + cat_lower = category.lower() + if any(c in cat_lower for c in critical_cats): + return "critical" if confidence > 0.7 else "high" + if any(c in cat_lower for c in high_cats): + return "high" if confidence > 0.7 else "medium" + return "medium" if confidence > 0.6 else "low" + + +def build_report( + url: str, + angles: list[dict], + screenshots: list[Path], + glitches: list[DetectedGlitch], +) -> ScanResult: + """Build the final structured scan report.""" + severity_counts = {} + category_counts = {} + + for g in glitches: + severity_counts[g.severity] = severity_counts.get(g.severity, 0) + 1 + category_counts[g.category] = category_counts.get(g.category, 0) + 1 + + report = ScanResult( + scan_id=str(uuid.uuid4()), + url=url, + timestamp=datetime.now(timezone.utc).isoformat(), + total_screenshots=len(screenshots), + angles_captured=[a["label"] for a in angles], + glitches=[asdict(g) for g in glitches], + summary={ + "total_glitches": len(glitches), + "by_severity": severity_counts, + "by_category": category_counts, + "highest_severity": max(severity_counts.keys(), default="none"), + "clean_screenshots": sum( + 1 + for i in range(len(screenshots)) + if not any(g.screenshot_index == i for g in glitches) + ), + }, + metadata={ + "detector_version": "0.1.0", + "pattern_count": len(MATRIX_GLITCH_PATTERNS), + "reference": "timmy-config#491", + }, + ) + + return report + + +def run_demo(output_path: Optional[Path] = None) -> ScanResult: + """Run a demonstration scan with simulated detections.""" + print("[*] Running Matrix glitch detection demo...") + + url = "https://matrix.example.com/world/alpha" + angles = generate_scan_angles(4) + screenshots_dir = Path("/tmp/matrix_glitch_screenshots") + + print(f"[*] Capturing {len(angles)} screenshots from: {url}") + screenshots = capture_screenshots(url, angles, screenshots_dir) + print(f"[*] Captured {len(screenshots)} screenshots") + + # Simulate detections for demo + demo_glitches = [ + DetectedGlitch( + id=str(uuid.uuid4())[:8], + category="floating_assets", + name="Floating Chair", + description="Office chair floating 0.3m above floor in sector 7", + severity="high", + confidence=0.87, + location_x=35.2, + location_y=62.1, + screenshot_index=0, + screenshot_angle="front", + ), + DetectedGlitch( + id=str(uuid.uuid4())[:8], + category="z_fighting", + name="Wall Texture Flicker", + description="Z-fighting between wall panel and decorative overlay", + severity="medium", + confidence=0.72, + location_x=58.0, + location_y=40.5, + screenshot_index=1, + screenshot_angle="right", + ), + DetectedGlitch( + id=str(uuid.uuid4())[:8], + category="missing_textures", + name="Placeholder Texture", + description="Bright magenta surface on door frame — missing asset reference", + severity="critical", + confidence=0.95, + location_x=72.3, + location_y=28.8, + screenshot_index=2, + screenshot_angle="back", + ), + DetectedGlitch( + id=str(uuid.uuid4())[:8], + category="clipping", + name="Desk Through Wall", + description="Desk corner clipping through adjacent wall geometry", + severity="high", + confidence=0.81, + location_x=15.0, + location_y=55.0, + screenshot_index=3, + screenshot_angle="left", + ), + ] + + print(f"[*] Detected {len(demo_glitches)} glitches") + report = build_report(url, angles, screenshots, demo_glitches) + + if output_path: + output_path.write_text(report.to_json()) + print(f"[*] Report saved to: {output_path}") + + return report + + +def main(): + parser = argparse.ArgumentParser( + description="Matrix 3D World Glitch Detector — scan for visual artifacts", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s https://matrix.example.com/world/alpha + %(prog)s https://matrix.example.com/world/alpha --angles 8 --output report.json + %(prog)s --demo + """, + ) + parser.add_argument("url", nargs="?", help="URL of the 3D world to scan") + parser.add_argument( + "--angles", type=int, default=4, help="Number of camera angles to capture (default: 4)" + ) + parser.add_argument("--output", "-o", type=str, help="Output file path for JSON report") + parser.add_argument("--demo", action="store_true", help="Run demo with simulated data") + parser.add_argument( + "--min-severity", + choices=["info", "low", "medium", "high", "critical"], + default="info", + help="Minimum severity to include in report", + ) + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + + args = parser.parse_args() + + if args.demo: + output = Path(args.output) if args.output else Path("glitch_report_demo.json") + report = run_demo(output) + print(f"\n=== Scan Summary ===") + print(f"URL: {report.url}") + print(f"Screenshots: {report.total_screenshots}") + print(f"Glitches found: {report.summary['total_glitches']}") + print(f"By severity: {report.summary['by_severity']}") + return + + if not args.url: + parser.error("URL required (or use --demo)") + + scan_id = str(uuid.uuid4())[:8] + print(f"[*] Matrix Glitch Detector — Scan {scan_id}") + print(f"[*] Target: {args.url}") + + # Generate camera angles + angles = generate_scan_angles(args.angles) + print(f"[*] Capturing {len(angles)} screenshots...") + + # Capture screenshots + screenshots_dir = Path(f"/tmp/matrix_glitch_{scan_id}") + screenshots = capture_screenshots(args.url, angles, screenshots_dir) + print(f"[*] Captured {len(screenshots)} screenshots") + + # Filter patterns by severity + min_sev = GlitchSeverity(args.min_severity) + patterns = get_patterns_by_severity(min_sev) + + # Analyze with vision AI + print(f"[*] Analyzing with vision AI ({len(patterns)} patterns)...") + glitches = analyze_with_vision(screenshots, angles, patterns) + + # Build and save report + report = build_report(args.url, angles, screenshots, glitches) + + if args.output: + Path(args.output).write_text(report.to_json()) + print(f"[*] Report saved: {args.output}") + else: + print(report.to_json()) + + print(f"\n[*] Done — {len(glitches)} glitches detected") + + +if __name__ == "__main__": + main() diff --git a/docs/glitch-detection.md b/docs/glitch-detection.md new file mode 100644 index 00000000..4fc0db67 --- /dev/null +++ b/docs/glitch-detection.md @@ -0,0 +1,179 @@ +# 3D World Glitch Detection — Matrix Scanner + +**Reference:** timmy-config#491 +**Label:** gemma-4-multimodal +**Version:** 0.1.0 + +## Overview + +The Matrix Glitch Detector scans 3D web worlds for visual artifacts and +rendering anomalies. It uses browser automation to capture screenshots from +multiple camera angles, then sends them to a vision AI model for analysis +against a library of known glitch patterns. + +## Detected Glitch Categories + +| Category | Severity | Description | +|---|---|---| +| Floating Assets | HIGH | Objects not grounded — hovering above surfaces | +| Z-Fighting | MEDIUM | Coplanar surfaces flickering/competing for depth | +| Missing Textures | CRITICAL | Placeholder colors (magenta, checkerboard) | +| Clipping | HIGH | Geometry passing through other objects | +| Broken Normals | MEDIUM | Inside-out or incorrectly lit surfaces | +| Shadow Artifacts | LOW | Detached, mismatched, or acne shadows | +| LOD Popping | LOW | Abrupt level-of-detail transitions | +| Lightmap Errors | MEDIUM | Dark splotches, light leaks, baking failures | +| Water/Reflection | MEDIUM | Incorrect environment reflections | +| Skybox Seam | LOW | Visible seams at cubemap face edges | + +## Installation + +No external dependencies required — pure Python 3.10+. + +```bash +# Clone the repo +git clone https://forge.alexanderwhitestone.com/Timmy_Foundation/timmy-config.git +cd timmy-config +``` + +## Usage + +### Basic Scan + +```bash +python bin/matrix_glitch_detector.py https://matrix.example.com/world/alpha +``` + +### Multi-Angle Scan + +```bash +python bin/matrix_glitch_detector.py https://matrix.example.com/world/alpha \ + --angles 8 \ + --output glitch_report.json +``` + +### Demo Mode + +```bash +python bin/matrix_glitch_detector.py --demo +``` + +### Options + +| Flag | Default | Description | +|---|---|---| +| `url` | (required) | URL of the 3D world to scan | +| `--angles N` | 4 | Number of camera angles to capture | +| `--output PATH` | stdout | Output file for JSON report | +| `--min-severity` | info | Minimum severity: info/low/medium/high/critical | +| `--demo` | off | Run with simulated detections | +| `--verbose` | off | Enable verbose output | + +## Report Format + +The JSON report includes: + +```json +{ + "scan_id": "uuid", + "url": "https://...", + "timestamp": "ISO-8601", + "total_screenshots": 4, + "angles_captured": ["front", "right", "back", "left"], + "glitches": [ + { + "id": "short-uuid", + "category": "floating_assets", + "name": "Floating Chair", + "description": "Office chair floating 0.3m above floor", + "severity": "high", + "confidence": 0.87, + "location_x": 35.2, + "location_y": 62.1, + "screenshot_index": 0, + "screenshot_angle": "front", + "timestamp": "ISO-8601" + } + ], + "summary": { + "total_glitches": 4, + "by_severity": {"critical": 1, "high": 2, "medium": 1}, + "by_category": {"floating_assets": 1, "missing_textures": 1, ...}, + "highest_severity": "critical", + "clean_screenshots": 0 + }, + "metadata": { + "detector_version": "0.1.0", + "pattern_count": 10, + "reference": "timmy-config#491" + } +} +``` + +## Vision AI Integration + +The detector supports any OpenAI-compatible vision API. Set these +environment variables: + +```bash +export VISION_API_KEY="your-api-key" +export VISION_API_BASE="https://api.openai.com/v1" # optional +export VISION_MODEL="gpt-4o" # optional, default: gpt-4o +``` + +For browser-based capture with `browser_vision`: + +```bash +export BROWSER_VISION_SCRIPT="/path/to/browser_vision.py" +``` + +## Glitch Patterns + +Pattern definitions live in `bin/glitch_patterns.py`. Each pattern includes: + +- **category** — Enum matching the glitch type +- **detection_prompts** — Instructions for the vision model +- **visual_indicators** — What to look for in screenshots +- **confidence_threshold** — Minimum confidence to report + +### Adding Custom Patterns + +```python +from glitch_patterns import GlitchPattern, GlitchCategory, GlitchSeverity + +custom = GlitchPattern( + category=GlitchCategory.FLOATING_ASSETS, + name="Custom Glitch", + description="Your description", + severity=GlitchSeverity.MEDIUM, + detection_prompts=["Look for..."], + visual_indicators=["indicator 1", "indicator 2"], +) +``` + +## Testing + +```bash +python -m pytest tests/test_glitch_detector.py -v +# or +python tests/test_glitch_detector.py +``` + +## Architecture + +``` +bin/ + matrix_glitch_detector.py — Main CLI entry point + glitch_patterns.py — Pattern definitions and prompt builder +tests/ + test_glitch_detector.py — Unit and integration tests +docs/ + glitch-detection.md — This documentation +``` + +## Limitations + +- Browser automation requires a headless browser environment +- Vision AI analysis depends on model availability and API limits +- Placeholder screenshots are generated when browser capture is unavailable +- Detection accuracy varies by scene complexity and lighting conditions diff --git a/tests/test_glitch_detector.py b/tests/test_glitch_detector.py new file mode 100644 index 00000000..970e529d --- /dev/null +++ b/tests/test_glitch_detector.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +""" +Tests for Matrix 3D Glitch Detector (timmy-config#491). + +Covers: glitch_patterns, matrix_glitch_detector core logic. +""" + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +# Ensure bin/ is importable +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "bin")) + +from glitch_patterns import ( + GlitchCategory, + GlitchPattern, + GlitchSeverity, + MATRIX_GLITCH_PATTERNS, + build_vision_prompt, + get_pattern_by_category, + get_patterns_by_severity, +) + +from matrix_glitch_detector import ( + DetectedGlitch, + ScanResult, + _infer_severity, + _parse_vision_response, + build_report, + generate_scan_angles, + run_demo, +) + + +class TestGlitchPatterns(unittest.TestCase): + """Tests for glitch_patterns module.""" + + def test_pattern_count(self): + """Verify we have a reasonable number of defined patterns.""" + self.assertGreaterEqual(len(MATRIX_GLITCH_PATTERNS), 8) + + def test_all_patterns_have_required_fields(self): + """Every pattern must have category, name, description, severity, prompts.""" + for p in MATRIX_GLITCH_PATTERNS: + self.assertIsInstance(p.category, GlitchCategory) + self.assertTrue(p.name) + self.assertTrue(p.description) + self.assertIsInstance(p.severity, GlitchSeverity) + self.assertGreater(len(p.detection_prompts), 0) + self.assertGreater(len(p.visual_indicators), 0) + self.assertGreater(p.confidence_threshold, 0) + self.assertLessEqual(p.confidence_threshold, 1.0) + + def test_pattern_to_dict(self): + """Pattern serialization should produce a dict with expected keys.""" + p = MATRIX_GLITCH_PATTERNS[0] + d = p.to_dict() + self.assertIn("category", d) + self.assertIn("name", d) + self.assertIn("severity", d) + self.assertEqual(d["category"], p.category.value) + + def test_get_patterns_by_severity(self): + """Severity filter should return only patterns at or above threshold.""" + high_patterns = get_patterns_by_severity(GlitchSeverity.HIGH) + self.assertTrue(all(p.severity.value in ("high", "critical") for p in high_patterns)) + self.assertGreater(len(high_patterns), 0) + + all_patterns = get_patterns_by_severity(GlitchSeverity.INFO) + self.assertEqual(len(all_patterns), len(MATRIX_GLITCH_PATTERNS)) + + def test_get_pattern_by_category(self): + """Lookup by category should return the correct pattern.""" + p = get_pattern_by_category(GlitchCategory.FLOATING_ASSETS) + self.assertIsNotNone(p) + self.assertEqual(p.category, GlitchCategory.FLOATING_ASSETS) + + missing = get_pattern_by_category("nonexistent_category_value") + self.assertIsNone(missing) + + def test_build_vision_prompt(self): + """Vision prompt should contain pattern names and be non-trivial.""" + prompt = build_vision_prompt() + self.assertGreater(len(prompt), 200) + self.assertIn("Floating Object", prompt) + self.assertIn("Z-Fighting", prompt) + self.assertIn("Missing", prompt) + + def test_build_vision_prompt_subset(self): + """Vision prompt with subset should only include specified patterns.""" + subset = MATRIX_GLITCH_PATTERNS[:3] + prompt = build_vision_prompt(subset) + self.assertIn(subset[0].name, prompt) + self.assertNotIn(MATRIX_GLITCH_PATTERNS[-1].name, prompt) + + +class TestGlitchDetector(unittest.TestCase): + """Tests for matrix_glitch_detector module.""" + + def test_generate_scan_angles_default(self): + """Default 4 angles should return front, right, back, left.""" + angles = generate_scan_angles(4) + self.assertEqual(len(angles), 4) + labels = [a["label"] for a in angles] + self.assertIn("front", labels) + self.assertIn("right", labels) + self.assertIn("back", labels) + self.assertIn("left", labels) + + def test_generate_scan_angles_many(self): + """Requesting more angles than base should still return correct count.""" + angles = generate_scan_angles(12) + self.assertEqual(len(angles), 12) + # Should still have the standard ones + labels = [a["label"] for a in angles] + self.assertIn("front", labels) + + def test_generate_scan_angles_few(self): + """Requesting fewer angles should return fewer.""" + angles = generate_scan_angles(2) + self.assertEqual(len(angles), 2) + + def test_detected_glitch_dataclass(self): + """DetectedGlitch should serialize cleanly.""" + g = DetectedGlitch( + id="test001", + category="floating_assets", + name="Test Glitch", + description="A test glitch", + severity="high", + confidence=0.85, + location_x=50.0, + location_y=30.0, + screenshot_index=0, + screenshot_angle="front", + ) + self.assertEqual(g.id, "test001") + self.assertTrue(g.timestamp) # Auto-generated + + def test_infer_severity_critical(self): + """Missing textures should infer critical/high severity.""" + sev = _infer_severity("missing_textures", 0.9) + self.assertEqual(sev, "critical") + sev_low = _infer_severity("missing_textures", 0.5) + self.assertEqual(sev_low, "high") + + def test_infer_severity_floating(self): + """Floating assets should infer high/medium severity.""" + sev = _infer_severity("floating_assets", 0.8) + self.assertEqual(sev, "high") + sev_low = _infer_severity("floating_assets", 0.5) + self.assertEqual(sev_low, "medium") + + def test_infer_severity_default(self): + """Unknown categories should default to medium/low.""" + sev = _infer_severity("unknown_thing", 0.7) + self.assertEqual(sev, "medium") + sev_low = _infer_severity("unknown_thing", 0.3) + self.assertEqual(sev_low, "low") + + def test_parse_vision_response_json_array(self): + """Should parse a JSON array response.""" + response = json.dumps([ + { + "category": "floating_assets", + "name": "Float Test", + "description": "Chair floating", + "confidence": 0.9, + "severity": "high", + "location_x": 40, + "location_y": 60, + } + ]) + glitches = _parse_vision_response(response, 0, "front") + self.assertEqual(len(glitches), 1) + self.assertEqual(glitches[0].category, "floating_assets") + self.assertAlmostEqual(glitches[0].confidence, 0.9) + + def test_parse_vision_response_wrapped(self): + """Should parse a response with 'glitches' wrapper key.""" + response = json.dumps({ + "glitches": [ + { + "category": "z_fighting", + "name": "Shimmer", + "confidence": 0.6, + } + ] + }) + glitches = _parse_vision_response(response, 1, "right") + self.assertEqual(len(glitches), 1) + self.assertEqual(glitches[0].category, "z_fighting") + + def test_parse_vision_response_empty(self): + """Should return empty list for non-JSON text.""" + glitches = _parse_vision_response("No glitches found.", 0, "front") + self.assertEqual(len(glitches), 0) + + def test_parse_vision_response_code_block(self): + """Should extract JSON from markdown code blocks.""" + response = '```json\n[{"category": "clipping", "name": "Clip", "confidence": 0.7}]\n```' + glitches = _parse_vision_response(response, 0, "front") + self.assertEqual(len(glitches), 1) + + def test_build_report(self): + """Report should have correct summary statistics.""" + angles = generate_scan_angles(4) + screenshots = [Path(f"/tmp/ss_{i}.png") for i in range(4)] + glitches = [ + DetectedGlitch( + id="a", category="floating_assets", name="Float", + description="", severity="high", confidence=0.8, + screenshot_index=0, screenshot_angle="front", + ), + DetectedGlitch( + id="b", category="missing_textures", name="Missing", + description="", severity="critical", confidence=0.95, + screenshot_index=1, screenshot_angle="right", + ), + ] + report = build_report("https://test.com", angles, screenshots, glitches) + + self.assertEqual(report.total_screenshots, 4) + self.assertEqual(len(report.glitches), 2) + self.assertEqual(report.summary["total_glitches"], 2) + self.assertEqual(report.summary["by_severity"]["critical"], 1) + self.assertEqual(report.summary["by_severity"]["high"], 1) + self.assertEqual(report.summary["by_category"]["floating_assets"], 1) + self.assertEqual(report.metadata["reference"], "timmy-config#491") + + def test_build_report_json_roundtrip(self): + """Report JSON should parse back correctly.""" + angles = generate_scan_angles(2) + screenshots = [Path(f"/tmp/ss_{i}.png") for i in range(2)] + report = build_report("https://test.com", angles, screenshots, []) + json_str = report.to_json() + parsed = json.loads(json_str) + self.assertEqual(parsed["url"], "https://test.com") + self.assertEqual(parsed["total_screenshots"], 2) + + def test_run_demo(self): + """Demo mode should produce a report with simulated glitches.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + output_path = Path(f.name) + + try: + report = run_demo(output_path) + self.assertEqual(len(report.glitches), 4) + self.assertGreater(report.summary["total_glitches"], 0) + self.assertTrue(output_path.exists()) + + # Verify the saved JSON is valid + saved = json.loads(output_path.read_text()) + self.assertIn("scan_id", saved) + self.assertIn("glitches", saved) + finally: + output_path.unlink(missing_ok=True) + + +class TestIntegration(unittest.TestCase): + """Integration-level tests.""" + + def test_full_pipeline_demo(self): + """End-to-end demo pipeline should complete without errors.""" + report = run_demo() + self.assertIsNotNone(report.scan_id) + self.assertTrue(report.timestamp) + self.assertGreater(report.total_screenshots, 0) + + def test_patterns_cover_matrix_themes(self): + """Patterns should cover the main Matrix glitch themes.""" + category_values = {p.category.value for p in MATRIX_GLITCH_PATTERNS} + expected = {"floating_assets", "z_fighting", "missing_textures", "clipping", "broken_normals"} + self.assertTrue(expected.issubset(category_values)) + + +if __name__ == "__main__": + unittest.main()