feat(10.8): add Progress Tracker — track test/doc coverage, issue close rate, dep freshness

Implements 10.8: Progress Tracker.

- Script: scripts/progress_tracker.py
- Stores weekly snapshots in metrics/snapshots/ (auto-created)
- Generates trends table in metrics/TRENDS.md
- Tracks: test-to-source ratio, doc coverage via AST, issue close rate (Gitea API), dep freshness (pip)
- Outputs stdout summary for weekly review

Usage:
    python3 scripts/progress_tracker.py
    python3 scripts/progress_tracker.py --json
    python3 scripts/progress_tracker.py --output metrics/TRENDS.md

Weekly cron: 0 9 * * 1 cd /path && python3 scripts/progress_tracker.py

Closes #173

scripts/github_trending_scanner.py

@@ -1,258 +0,0 @@
-#!/usr/bin/env python3
-"""GitHub Trending Scanner — Scan trending repos in AI/ML.
-
-Extracts: repo description, stars, key features (topics, inferred highlights).
-Filters by language and/or topic. Outputs dated JSON for daily scan pipeline.
-
-Usage:
-    python3 github_trending_scanner.py --language python --topic ai --output metrics/trending
-    python3 github_trending_scanner.py --topic machine-learning --limit 50
-    python3 github_trending_scanner.py --language rust --topic artificial-intelligence
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Optional, List, Dict
-import urllib.request
-import urllib.parse
-import urllib.error
-
-GITHUB_API_BASE = os.environ.get("GITHUB_API_BASE", "https://api.github.com")
-DEFAULT_OUTPUT_DIR = os.environ.get("TRENDING_OUTPUT_DIR", "metrics/trending")
-DEFAULT_LIMIT = int(os.environ.get("TRENDING_LIMIT", "30"))
-DEFAULT_MIN_STARS = int(os.environ.get("TRENDING_MIN_STARS", "1000"))
-
-
-def fetch_trending_repos(
-    language: Optional[str] = None,
-    topic: Optional[str] = None,
-    min_stars: int = DEFAULT_MIN_STARS,
-    limit: int = DEFAULT_LIMIT,
-) -> List[Dict]:
-    """Fetch trending-like repositories from GitHub using the search API.
-
-    GitHub's public search API is unauthenticated-rate-limited (60 req/hr).
-    This function retries on rate-limit backoff and falls back gracefully.
-    """
-    # Build search query: stars threshold + optional language/topic filters
-    query = f"stars:>{min_stars}"
-    if language:
-        query += f" language:{language}"
-    if topic:
-        query += f" topic:{topic}"
-
-    # Sort by stars descending as a proxy for trending/popular
-    params = {
-        "q": query,
-        "sort": "stars",
-        "order": "desc",
-        "per_page": min(limit, 100),  # GitHub max per_page is 100
-    }
-    url = f"{GITHUB_API_BASE}/search/repositories?{urllib.parse.urlencode(params)}"
-
-    headers = {
-        "Accept": "application/vnd.github.v3+json",
-        "User-Agent": "Sovereign-Trending-Scanner/1.0",
-    }
-
-    for attempt in range(3):
-        try:
-            req = urllib.request.Request(url, headers=headers)
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                if resp.status != 200:
-                    raise RuntimeError(f"GitHub API returned {resp.status}")
-                data = json.loads(resp.read().decode("utf-8"))
-                return data.get("items", [])[:limit]
-        except urllib.error.HTTPError as e:
-            if e.code == 403:
-                # Check for rate limit message
-                body = e.read().decode("utf-8", errors="replace").lower()
-                if "rate limit" in body or "api rate limit exceeded" in body:
-                    reset_ts = int(e.headers.get("X-RateLimit-Reset", 0))
-                    wait_seconds = max(5, reset_ts - int(time.time()) + 5)
-                    print(f"Rate limit exceeded — waiting {wait_seconds}s (attempt {attempt+1}/3)...", file=sys.stderr)
-                    time.sleep(wait_seconds)
-                    continue
-            print(f"ERROR: GitHub API request failed: {e} — {e.read().decode('utf-8', errors='replace')[:200]}", file=sys.stderr)
-            return []
-        except Exception as e:
-            if attempt < 2:
-                backoff = 2 ** attempt
-                print(f"WARNING: Fetch attempt {attempt+1} failed: {e} — retrying in {backoff}s", file=sys.stderr)
-                time.sleep(backoff)
-                continue
-            print(f"ERROR: All fetch attempts failed: {e}", file=sys.stderr)
-            return []
-
-    return []
-
-
-def extract_repo_features(repo_data: Dict) -> Dict:
-    """Extract structured fields for a trending repo."""
-    description = (repo_data.get("description") or "").strip()
-    topics = repo_data.get("topics", [])
-
-    # Infer key features from description and topics
-    features = infer_features(description, topics)
-
-    return {
-        "name": repo_data.get("full_name", ""),
-        "description": description,
-        "stars": repo_data.get("stargazers_count", 0),
-        "forks": repo_data.get("forks_count", 0),
-        "open_issues": repo_data.get("open_issues_count", 0),
-        "language": repo_data.get("language", ""),
-        "topics": topics,
-        "url": repo_data.get("html_url", ""),
-        "created_at": repo_data.get("created_at", ""),
-        "updated_at": repo_data.get("updated_at", ""),
-        "key_features": features,
-        "scanned_at": datetime.now(timezone.utc).isoformat(),
-    }
-
-
-def infer_features(description: str, topics: List[str]) -> List[str]:
-    """Infer notable capabilities/features from repo metadata.
-
-    Looks for AI/ML-relevant capabilities in topics and description.
-    """
-    features = []
-    text = (description + " " + " ".join(topics)).lower()
-
-    # Domain capabilities (keys normalized to lowercase for consistency)
-    capability_keywords = {
-        "fine-tuning": ["fine-tun", "finetun"],
-        "agent framework": ["agent"],
-        "local/offline": ["local", "on-device", "offline"],
-        "quantized models": ["quantized", "quantization", "gguf", "gptq"],
-        "vision": ["vision", "multimodal", "image", "visual"],
-        "speech/audio": ["speech", "audio", "whisper", "tts"],
-        "retrieval/rag": ["rag", "retrieval", "embedding", "vector"],
-        "training": ["train", "training", "sft", "dpo"],
-        "gui/playground": ["gui", "playground", "webui", "interface"],
-        "sota": ["state-of-the-art", "sota", "latest"],
-    }
-
-    for label, keywords in capability_keywords.items():
-        if any(kw in text for kw in keywords):
-            features.append(label)
-
-    # Also include non-generic topics as features
-    generic_topics = {"ai", "ml", "machine-learning", "deep-learning", "llm", "python", "pytorch", "tensorflow"}
-    for topic in topics:
-        if topic.lower() not in generic_topics:
-            features.append(topic)
-
-    # Deduplicate while preserving order, return up to 10
-    seen = set()
-    unique = []
-    for f in features:
-        key = f.lower()
-        if key not in seen:
-            seen.add(key)
-            unique.append(f)
-    return unique[:10]
-
-
-def save_trending(repos: List[Dict], output_dir: str = "metrics/trending") -> str:
-    """Save trending results to a dated JSON file.
-
-    Returns the path of the written file.
-    """
-    output_path = Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    filename = output_path / f"github-trending-{date_str}.json"
-
-    output_data = {
-        "scanned_at": datetime.now(timezone.utc).isoformat(),
-        "count": len(repos),
-        "repos": repos,
-    }
-
-    with open(filename, "w") as f:
-        json.dump(output_data, f, indent=2, ensure_ascii=False)
-
-    return str(filename)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Scan GitHub trending repositories in AI/ML"
-    )
-    parser.add_argument(
-        "--language",
-        help="Filter by programming language (e.g., python, rust, go)",
-    )
-    parser.add_argument(
-        "--topic",
-        help="Filter by GitHub topic (e.g., ai, machine-learning, llm)",
-    )
-    parser.add_argument(
-        "--since",
-        default="daily",
-        choices=["daily", "weekly", "monthly"],
-        help="Trending period (daily/weekly/monthly) — informational only",
-    )
-    parser.add_argument(
-        "--output",
-        default="metrics/trending",
-        help="Output directory for results (default: metrics/trending)",
-    )
-    parser.add_argument(
-        "--limit",
-        type=int,
-        default=DEFAULT_LIMIT,
-        help=f"Maximum repos to fetch (default: {DEFAULT_LIMIT})",
-    )
-    parser.add_argument(
-        "--min-stars",
-        type=int,
-        default=DEFAULT_MIN_STARS,
-        help=f"Minimum star count for relevance (default: {DEFAULT_MIN_STARS})",
-    )
-    args = parser.parse_args()
-
-    print(
-        f"Fetching trending repos "
-        f"(language={args.language or 'any'}, topic={args.topic or 'any'}, period={args.since})..."
-    )
-
-    repos_raw = fetch_trending_repos(
-        language=args.language,
-        topic=args.topic,
-        min_stars=args.min_stars,
-        limit=args.limit,
-    )
-
-    if not repos_raw:
-        print("WARNING: No repos fetched — check network or rate limits", file=sys.stderr)
-
-    repos = [extract_repo_features(r) for r in repos_raw]
-
-    output_file = save_trending(repos, args.output)
-    print(f"Saved {len(repos)} trending repos to {output_file}")
-
-    # Brief human-readable summary
-    if repos:
-        print("\nTop repos:")
-        for repo in repos[:5]:
-            features_preview = ", ".join(repo["key_features"][:3])
-            print(f"  ★ {repo['stars']:>7}  {repo['name']}")
-            if repo["description"]:
-                desc = repo["description"][:80]
-                print(f"         {desc}{'...' if len(repo['description']) > 80 else ''}")
-            if features_preview:
-                print(f"         Features: {features_preview}")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/progress_tracker.py

@@ -0,0 +1,477 @@
+#!/usr/bin/env python3
+"""
+Progress Tracker — Pipeline 10.8
+Track improvement metrics over time. Are we getting better?
+
+Metrics tracked:
+ 1. Test coverage  — % of Python functions with associated tests (test:source file ratio + line coverage if available)
+ 2. Doc coverage   — % of Python callables with docstrings (AST-based)
+ 3. Issue close rate — closed / (opened + closed) per week (Gitea API)
+ 4. Dep freshness  — % of requirements pinned vs outdated (pip list --outdated)
+
+Output:
+ - metrics/snapshots/YYYY-MM-DD.json  — one snapshot per run
+ - metrics/TRENDS.md                  — cumulative markdown table
+ - stdout summary
+
+Usage:
+    python3 scripts/progress_tracker.py
+    python3 scripts/progress_tracker.py --json
+    python3 scripts/progress_tracker.py --output metrics/TRENDS.md
+
+Weekly cron:
+    0 9 * * 1 cd /path/to/compounding-intelligence && python3 scripts/progress_tracker.py
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+# ── Configuration ──────────────────────────────────────────────────────────
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parent
+METRICS_DIR = REPO_ROOT / "metrics"
+SNAPSHOTS_DIR = METRICS_DIR / "snapshots"
+TOKEN_PATH = Path.home() / ".config" / "gitea" / "token"
+GITEA_API_BASE = "https://forge.alexanderwhitestone.com/api/v1"
+ORG = "Timmy_Foundation"
+
+# Ensure paths exist
+SNAPSHOTS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────────
+
+def run_cmd(cmd: List[str], cwd: Path = REPO_ROOT) -> str:
+    """Run a shell command and return stdout (stderr merged)."""
+    result = subprocess.run(
+        cmd, capture_output=True, text=True, cwd=cwd, timeout=30
+    )
+    if result.returncode != 0:
+        return ""
+    return result.stdout.strip()
+
+
+def slugify_date(dt: datetime) -> str:
+    return dt.strftime("%Y-%m-%d")
+
+
+def snapshot_path(dt: datetime) -> Path:
+    return SNAPSHOTS_DIR / f"{slugify_date(dt)}.json"
+
+
+def load_snapshots() -> List[Dict[str, Any]]:
+    """Load all existing snapshots sorted by date."""
+    snapshots = []
+    for f in sorted(SNAPSHOTS_DIR.glob("*.json")):
+        try:
+            with open(f) as fp:
+                snapshots.append(json.load(fp))
+        except Exception:
+            continue
+    return snapshots
+
+
+# ── Metric 1: Test Coverage ─────────────────────────────────────────────────
+
+def collect_test_coverage() -> Dict[str, Any]:
+    """
+    Compute test coverage metrics.
+    Counts test_*.py and *_test.py files vs non-test .py source files.
+    Also attempts to read .coverage if present.
+    """
+    all_py = list(REPO_ROOT.rglob("*.py"))
+    
+    source_files = []
+    test_files = []
+    
+    for p in all_py:
+        try:
+            rel_parts = p.relative_to(REPO_ROOT).parts
+        except ValueError:
+            continue
+        
+        # Skip hidden/cache/temp dirs (check only relative parts)
+        if any(part.startswith('.') or part.startswith('__') for part in rel_parts):
+            continue
+        if any(part in ('node_modules', 'venv', '.venv', 'env', '.pytest_cache') for part in rel_parts):
+            continue
+        
+        if p.name.startswith("test_") or p.name.endswith("_test.py"):
+            test_files.append(p)
+        else:
+            source_files.append(p)
+    
+    # Try to get line coverage from .coverage
+    coverage_percent = None
+    coverage_tool = None
+    coverage_file = REPO_ROOT / ".coverage"
+    if coverage_file.exists():
+        try:
+            import coverage  # type: ignore
+            # Use coverage API if available
+            cov = coverage.Coverage(data_file=str(coverage_file))
+            cov.load()
+            total = cov.report()
+            coverage_percent = total if isinstance(total, float) else None
+            coverage_tool = "coverage"
+        except Exception:
+            # Fallback: parse `coverage report` output
+            out = run_cmd(["coverage", "report", "--skip-empty"])
+            if out:
+                for line in out.splitlines():
+                    if "TOTAL" in line:
+                        parts = line.split()
+                        if len(parts) >= 2:
+                            try:
+                                coverage_percent = float(parts[-1].rstrip('%'))
+                                coverage_tool = "coverage"
+                                break
+                            except ValueError:
+                                pass
+    
+    return {
+        "test_files": len(test_files),
+        "source_files": len(source_files),
+        "test_to_source_ratio": round(len(test_files) / len(source_files), 4) if source_files else 0.0,
+        "coverage_tool": coverage_tool,
+        "coverage_percent": coverage_percent,
+    }
+
+
+# ── Metric 2: Doc Coverage ──────────────────────────────────────────────────
+
+def collect_doc_coverage() -> Dict[str, Any]:
+    """
+    Check AST of Python files for docstrings.
+    Returns: callables_total, callables_with_doc, doc_coverage_percent
+    """
+    import ast
+    
+    all_py = list(REPO_ROOT.rglob("*.py"))
+    
+    source_files = []
+    test_files = []
+    
+    for p in all_py:
+        try:
+            rel_parts = p.relative_to(REPO_ROOT).parts
+        except ValueError:
+            continue
+        
+        if any(part.startswith('.') or part.startswith('__') for part in rel_parts):
+            continue
+        if any(part in ('node_modules', 'venv', '.venv', 'env', '.pytest_cache') for part in rel_parts):
+            continue
+        
+        if p.name.startswith("test_") or p.name.endswith("_test.py"):
+            test_files.append(p)
+        else:
+            source_files.append(p)
+    
+    total_callables = 0
+    with_doc = 0
+    
+    for p in source_files + test_files:
+        try:
+            with open(p) as f:
+                tree = ast.parse(f.read(), filename=str(p))
+            for node in ast.walk(tree):
+                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                    total_callables += 1
+                    doc = ast.get_docstring(node)
+                    if doc and doc.strip():
+                        with_doc += 1
+        except Exception:
+            continue
+    
+    return {
+        "callables_total": total_callables,
+        "callables_with_doc": with_doc,
+        "doc_coverage_percent": round((with_doc / total_callables * 100) if total_callables else 0.0, 2),
+    }
+
+
+# ── Metric 3: Issue Close Rate ──────────────────────────────────────────────
+
+def collect_issue_metrics() -> Dict[str, Any]:
+    """
+    Use Gitea API to get issue open/close stats for the last 7 days.
+    Returns counts and close rate.
+    """
+    token = ""
+    if TOKEN_PATH.exists():
+        token = TOKEN_PATH.read_text().strip()
+    
+    if not token:
+        return {
+            "opened_last_7d": None,
+            "closed_last_7d": None,
+            "close_rate": None,
+            "total_open": None,
+            "note": "Gitea token not available"
+        }
+    
+    try:
+        from urllib.request import Request, urlopen
+        from urllib.error import HTTPError, URLError
+    except ImportError:
+        return {"error": "urllib not available"}
+    
+    now = datetime.now(timezone.utc)
+    week_ago = now - timedelta(days=7)
+    since = week_ago.strftime("%Y-%m-%d")
+    
+    headers = {"Authorization": f"token {token}"}
+    base_url = f"{GITEA_API_BASE}/repos/{ORG}/compounding-intelligence/issues"
+    
+    try:
+        # Get issues from last 7 days
+        url = f"{base_url}?state=all&since={since}&per_page=100"
+        req = Request(url, headers=headers)
+        with urlopen(req, timeout=15) as resp:
+            issues = json.loads(resp.read())
+        
+        opened = 0
+        closed = 0
+        for issue in issues:
+            created = datetime.fromisoformat(issue["created_at"].replace("Z", "+00:00"))
+            if created >= week_ago:
+                opened += 1
+            if issue.get("state") == "closed":
+                closed_at_str = issue.get("closed_at")
+                if closed_at_str:
+                    closed_at = datetime.fromisoformat(closed_at_str.replace("Z", "+00:00"))
+                    if closed_at >= week_ago:
+                        closed += 1
+        
+        # Total open issues
+        req2 = Request(f"{base_url}?state=open&per_page=1", headers=headers)
+        with urlopen(req2, timeout=15) as resp:
+            total_open = int(resp.headers.get("X-Total-Count", "0"))
+        
+        total = opened + closed
+        close_rate = closed / total if total > 0 else 0.0
+        
+        return {
+            "opened_last_7d": opened,
+            "closed_last_7d": closed,
+            "close_rate": round(close_rate, 4),
+            "total_open": total_open,
+        }
+    except Exception as e:
+        return {
+            "opened_last_7d": None,
+            "closed_last_7d": None,
+            "close_rate": None,
+            "total_open": None,
+            "error": str(e)[:100],
+            "note": "Gitea API unavailable"
+        }
+
+
+# ── Metric 4: Dependency Freshness ─────────────────────────────────────────
+
+def collect_dep_freshness() -> Dict[str, Any]:
+    """
+    Check requirements.txt for outdated dependencies using pip list --outdated.
+    Returns freshness percentage and outdated list.
+    """
+    req_file = REPO_ROOT / "requirements.txt"
+    if not req_file.exists():
+        return {
+            "total_deps": 0,
+            "outdated_deps": 0,
+            "freshness_percent": 100.0,
+            "outdated_list": [],
+            "note": "requirements.txt not found"
+        }
+    
+    # Parse requirements (very simple: take name before comparison op)
+    reqs = []
+    with open(req_file) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            m = re.match(r"^([a-zA-Z0-9_.-]+)", line)
+            if m:
+                reqs.append(m.group(1))
+    
+    if not reqs:
+        return {"total_deps": 0, "outdated_deps": 0, "freshness_percent": 100.0, "outdated_list": []}
+    
+    # Query pip for outdated packages (may fail if pip not available)
+    outdated_names = set()
+    try:
+        out = run_cmd(["pip", "list", "--outdated", "--format=json"])
+        if out:
+            data = json.loads(out)
+            outdated_names = {item["name"].lower() for item in data}
+    except Exception:
+        pass
+    
+    outdated = [p for p in reqs if p.lower() in outdated_names]
+    total = len(reqs)
+    outdated_count = len(outdated)
+    freshness = round(((total - outdated_count) / total * 100) if total else 100.0, 1)
+    
+    return {
+        "total_deps": total,
+        "outdated_deps": outdated_count,
+        "freshness_percent": freshness,
+        "outdated_list": outdated,
+    }
+
+
+# ── Snapshot & Trends ───────────────────────────────────────────────────────
+
+def take_snapshot() -> Dict[str, Any]:
+    """Collect all metrics and return a snapshot dict."""
+    now = datetime.now(timezone.utc)
+    test_cov = collect_test_coverage()
+    doc_cov = collect_doc_coverage()
+    issues = collect_issue_metrics()
+    deps = collect_dep_freshness()
+    
+    return {
+        "timestamp": now.isoformat(),
+        "date": slugify_date(now),
+        "metrics": {
+            "test_coverage": test_cov,
+            "doc_coverage": doc_cov,
+            "issues": issues,
+            "dependencies": deps,
+        }
+    }
+
+
+def save_snapshot(snapshot: Dict[str, Any]) -> Path:
+    path = snapshot_path(datetime.fromisoformat(snapshot["timestamp"]))
+    with open(path, "w") as f:
+        json.dump(snapshot, f, indent=2)
+    return path
+
+
+def generate_trends(snapshots: List[Dict[str, Any]], output_path: Optional[Path] = None) -> str:
+    """Generate markdown trends table; optionally write to file."""
+    if not snapshots:
+        msg = "# Progress Tracker — Trends\n\nNo snapshots yet. Run `progress_tracker.py` to create the first snapshot."
+        if output_path:
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            output_path.write_text(msg)
+        return msg
+    
+    lines = [
+        "# Progress Tracker — Trends",
+        f"\nLast updated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
+        f"\nSnapshots: {len(snapshots)}\n",
+        "| Date | Test Files → Source | Doc Coverage | Issues Closed/Opened (7d) | Dep Freshness |",
+        "|------|---------------------|--------------|---------------------------|---------------|",
+    ]
+    
+    for snap in reversed(snapshots):  # chronological
+        date = snap["date"]
+        m = snap["metrics"]
+        tc = m["test_coverage"]
+        test_str = f"{tc['test_files']}/{tc['source_files']} ({tc['test_to_source_ratio']:.2f})"
+        doc_str = f"{m['doc_coverage']['doc_coverage_percent']:.1f}%"
+        issues_str = f"{m['issues'].get('closed_last_7d','-')}/{m['issues'].get('opened_last_7d','-')}"
+        dep_str = f"{m['dependencies'].get('freshness_percent','?')}%"
+        lines.append(f"| {date} | {test_str} | {doc_str} | {issues_str} | {dep_str} |")
+    
+    # Current snapshot summary
+    cur = snapshots[-1]
+    cm = cur["metrics"]
+    lines.append(f"\n## Current Snapshot ({cur['date']})\n")
+    
+    tc = cm["test_coverage"]
+    cov_line = f"- Test coverage: {tc['coverage_percent']:.1f}% (via {tc['coverage_tool']})\n" if tc["coverage_percent"] else "- Test coverage: (pytest-cov not configured)\n"
+    lines.append(cov_line)
+    lines.append(f"- Doc coverage: {cm['doc_coverage']['doc_coverage_percent']:.1f}%")
+    
+    im = cm["issues"]
+    if im.get("close_rate") is not None:
+        lines.append(f"- Issue close rate (7d): {im['close_rate']*100:.1f}% ({im['closed_last_7d']} closed, {im['opened_last_7d']} opened)")
+    else:
+        lines.append(f"- Issue metrics: {im.get('note','unavailable')}")
+    
+    dd = cm["dependencies"]
+    lines.append(f"- Dep freshness: {dd.get('freshness_percent','?')}% outdated ({dd.get('outdated_deps',0)}/{dd.get('total_deps',0)} deps)")
+    if dd.get('outdated_list'):
+        lines.append(f"  Outdated: {', '.join(dd['outdated_list'][:5])}")
+    
+    content = "\n".join(lines) + "\n"
+    
+    if output_path:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(content)
+    
+    return content
+
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Progress Tracker — 10.8")
+    parser.add_argument("--json", action="store_true", help="Emit snapshot as JSON only")
+    parser.add_argument("--output", type=Path, default=METRICS_DIR / "TRENDS.md",
+                        help="Write trends markdown to this file")
+    args = parser.parse_args()
+    
+    snapshot = take_snapshot()
+    all_snapshots = load_snapshots()
+    path_written = save_snapshot(snapshot)
+    
+    if args.json:
+        print(json.dumps(snapshot, indent=2))
+        return 0
+    
+    trends = generate_trends(all_snapshots + [snapshot], output_path=args.output)
+    
+    # Print current snapshot summary
+    print(f"Snapshot saved: {path_written}\n")
+    print(f"Progress Tracker — {snapshot['date']}")
+    print("=" * 50)
+    
+    m = snapshot["metrics"]
+    tc = m["test_coverage"]
+    print(f"Test files: {tc['test_files']}  |  Source files: {tc['source_files']}  |  Ratio: {tc['test_to_source_ratio']:.3f}")
+    if tc["coverage_percent"] is not None:
+        print(f"Line coverage: {tc['coverage_percent']:.1f}%  (via {tc['coverage_tool']})")
+    else:
+        print("Line coverage: (not available — run `pytest --cov`)")
+    
+    print()
+    dc = m["doc_coverage"]
+    print(f"Callables with docstrings: {dc['callables_with_doc']}/{dc['callables_total']}  ({dc['doc_coverage_percent']:.1f}%)")
+    
+    print()
+    im = m["issues"]
+    if im.get("close_rate") is not None:
+        print(f"Issues (7d): {im['closed_last_7d']} closed / {im['opened_last_7d']} opened  → close rate: {im['close_rate']*100:.1f}%")
+        print(f"Total open: {im['total_open']}")
+    else:
+        print(f"Issues: {im.get('note','unavailable')}")
+    
+    print()
+    dd = m["dependencies"]
+    print(f"Dependencies: {dd.get('total_deps',0)} total, {dd.get('outdated_deps',0)} outdated")
+    if dd.get('outdated_list'):
+        shown = dd['outdated_list'][:5]
+        print(f"Outdated: {', '.join(shown)}" + ("..." if len(dd['outdated_list']) > 5 else ""))
+    
+    print(f"\nTrends written to: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/test_github_trending_scanner.py

@@ -1,125 +0,0 @@
-#!/usr/bin/env python3
-"""Tests for github_trending_scanner.py — pure function validation.
-
-Tests the feature inference, extraction, and output formatting logic
-without relying on external GitHub API calls.
-"""
-
-import json
-import sys
-import tempfile
-from pathlib import Path
-
-# Add scripts dir to path for import
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-
-from github_trending_scanner import (
-    extract_repo_features,
-    infer_features,
-    save_trending,
-)
-
-
-def test_infer_features_from_description():
-    """Feature inference extracts capabilities from description text."""
-    desc = "A local, quantized LLM framework for fine-tuning and agent-based RAG with vision."
-    topics = ["ai", "llm"]
-    features = infer_features(desc, topics)
-
-    # Should include relevant capabilities (case-insensitive comparison)
-    expected_lower = {"fine-tuning", "local/offline", "quantized models", "agent framework", "vision", "retrieval/rag"}
-    actual_lower = set(f.lower() for f in features)
-    assert expected_lower.issubset(actual_lower), f"Missing features. Expected subset of {expected_lower}, got {actual_lower}"
-    print("PASS: infer_features_from_description")
-
-
-def test_infer_features_from_topics_only():
-    """Topics alone can drive feature detection."""
-    desc = ""
-    topics = ["computer-vision", "speech", "pytorch"]
-    features = infer_features(desc, topics)
-
-    # Non-generic topics should appear as features (topics preserved as-is)
-    assert "computer-vision" in features, f"Expected 'computer-vision' in {features}"
-    assert "speech" in features, f"Expected 'speech' in {features}"
-    # Generic topics (pytorch) may be filtered
-    print(f"PASS: infer_features_from_topics_only → {features}")
-
-
-def test_extract_repo_features_produces_valid_structure():
-    """extract_repo_features returns all required fields."""
-    mock_repo = {
-        "full_name": "example/repo",
-        "description": "An example repository",
-        "stargazers_count": 1234,
-        "forks_count": 56,
-        "open_issues_count": 7,
-        "language": "Python",
-        "topics": ["ai", "llm"],
-        "html_url": "https://github.com/example/repo",
-        "created_at": "2025-01-01T00:00:00Z",
-        "updated_at": "2026-01-01T00:00:00Z",
-    }
-
-    result = extract_repo_features(mock_repo)
-
-    assert result["name"] == "example/repo"
-    assert result["description"] == "An example repository"
-    assert result["stars"] == 1234
-    assert isinstance(result["key_features"], list)
-    assert "scanned_at" in result
-    assert result["url"] == "https://github.com/example/repo"
-    print("PASS: extract_repo_features_structure")
-
-
-def test_save_trending_creates_dated_json():
-    """save_trending writes a valid JSON file with the expected schema."""
-    repos = [
-        {
-            "name": "test/repo",
-            "description": "Test repository",
-            "stars": 999,
-            "language": "Python",
-            "topics": ["test"],
-            "key_features": ["testing"],
-            "scanned_at": "2026-04-26T00:00:00+00:00",
-        }
-    ]
-
-    with tempfile.TemporaryDirectory() as tmp:
-        output_file = save_trending(repos, output_dir=tmp)
-
-        path = Path(output_file)
-        assert path.exists(), f"Output file not created: {output_file}"
-
-        with open(path) as f:
-            data = json.load(f)
-
-        assert "scanned_at" in data
-        assert data["count"] == 1
-        assert isinstance(data["repos"], list)
-        assert data["repos"][0]["name"] == "test/repo"
-        print(f"PASS: save_trending → {output_file}")
-
-
-def test_save_trending_respects_output_dir_creation():
-    """Output directory is created if it doesn't exist."""
-    repos = []
-
-    with tempfile.TemporaryDirectory() as tmp:
-        nested = Path(tmp) / "nested" / "trending"
-        assert not nested.exists()
-
-        output_file = save_trending(repos, output_dir=str(nested))
-        assert nested.exists()
-        assert Path(output_file).exists()
-        print("PASS: output_dir_creation")
-
-
-if __name__ == "__main__":
-    test_infer_features_from_description()
-    test_infer_features_from_topics_only()
-    test_extract_repo_features_produces_valid_structure()
-    test_save_trending_creates_dated_json()
-    test_save_trending_respects_output_dir_creation()
-    print("\nAll github_trending_scanner tests passed.")