feat: knowledge store staleness detector (#92)

scripts/knowledge_staleness_check.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""
+Knowledge Store Staleness Detector
+
+Checks knowledge entries against their source files to detect staleness.
+An entry is stale when its source file has been modified since extraction.
+
+Usage:
+  python3 scripts/knowledge_staleness_check.py knowledge/index.json
+  python3 scripts/knowledge_staleness_check.py --repo /path/to/repo --index knowledge/index.json
+  python3 scripts/knowledge_staleness_check.py --index knowledge/index.json --fix
+
+Expected index.json format:
+{
+  "version": 1,
+  "facts": [
+    {
+      "fact": "...",
+      "category": "fact|pitfall|pattern|tool-quirk",
+      "repo": "repo-name",
+      "confidence": 0.8,
+      "source_file": "path/to/file.py",
+      "source_hash": "sha256:abcdef...",
+      "extracted_at": "2026-04-13T20:00:00Z"
+    }
+  ]
+}
+"""
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+
+def compute_file_hash(filepath: str) -> Optional[str]:
+    """Compute SHA-256 hash of a file. Returns None if file not found."""
+    path = Path(filepath)
+    if not path.exists():
+        return None
+    content = path.read_bytes()
+    return hashlib.sha256(content).hexdigest()[:16]
+
+
+def check_staleness(index_path: str, repo_root: str = None) -> dict:
+    """Check all entries in the knowledge index for staleness."""
+    index = Path(index_path)
+    if not index.exists():
+        return {"error": f"Index not found: {index_path}"}
+
+    data = json.loads(index.read_text())
+    facts = data.get("facts", [])
+
+    if not facts:
+        return {
+            "total": 0,
+            "stale": 0,
+            "fresh": 0,
+            "no_source": 0,
+            "missing_files": 0,
+            "stale_entries": [],
+        }
+
+    # Determine repo root
+    if repo_root:
+        root = Path(repo_root)
+    else:
+        root = index.parent.parent  # knowledge/index.json -> repo root
+
+    results = {
+        "total": len(facts),
+        "stale": 0,
+        "fresh": 0,
+        "no_source": 0,
+        "missing_files": 0,
+        "stale_entries": [],
+    }
+
+    for i, entry in enumerate(facts):
+        source_file = entry.get("source_file")
+        stored_hash = entry.get("source_hash")
+
+        if not source_file:
+            results["no_source"] += 1
+            continue
+
+        if not stored_hash:
+            # Entry has source file but no hash — consider stale
+            results["stale"] += 1
+            results["stale_entries"].append({
+                "index": i,
+                "fact": entry.get("fact", "")[:100],
+                "source_file": source_file,
+                "reason": "no_hash",
+            })
+            continue
+
+        # Compute current hash
+        full_path = root / source_file
+        current_hash = compute_file_hash(str(full_path))
+
+        if current_hash is None:
+            results["missing_files"] += 1
+            results["stale_entries"].append({
+                "index": i,
+                "fact": entry.get("fact", "")[:100],
+                "source_file": source_file,
+                "reason": "file_missing",
+            })
+        elif current_hash != stored_hash:
+            results["stale"] += 1
+            results["stale_entries"].append({
+                "index": i,
+                "fact": entry.get("fact", "")[:100],
+                "source_file": source_file,
+                "stored_hash": stored_hash,
+                "current_hash": current_hash,
+                "reason": "hash_mismatch",
+            })
+        else:
+            results["fresh"] += 1
+
+    return results
+
+
+def add_hashes_to_index(index_path: str, repo_root: str = None) -> dict:
+    """Add source hashes to entries that are missing them."""
+    index = Path(index_path)
+    data = json.loads(index.read_text())
+    facts = data.get("facts", [])
+
+    if repo_root:
+        root = Path(repo_root)
+    else:
+        root = index.parent.parent
+
+    updated = 0
+    skipped = 0
+
+    for entry in facts:
+        source_file = entry.get("source_file")
+        if not source_file or entry.get("source_hash"):
+            skipped += 1
+            continue
+
+        full_path = root / source_file
+        file_hash = compute_file_hash(str(full_path))
+        if file_hash:
+            entry["source_hash"] = file_hash
+            updated += 1
+
+    if updated > 0:
+        index.write_text(json.dumps(data, indent=2) + "\n")
+
+    return {"updated": updated, "skipped": skipped, "total": len(facts)}
+
+
+def report_staleness(results: dict) -> str:
+    """Format staleness check results as a report."""
+    lines = []
+    lines.append("=" * 50)
+    lines.append("KNOWLEDGE STORE STALENESS REPORT")
+    lines.append("=" * 50)
+    lines.append(f"Total entries:  {results['total']}")
+    lines.append(f"Fresh:          {results['fresh']}")
+    lines.append(f"Stale:          {results['stale']}")
+    lines.append(f"No source:      {results['no_source']}")
+    lines.append(f"Missing files:  {results['missing_files']}")
+    lines.append("")
+
+    if results["stale_entries"]:
+        lines.append("STALE ENTRIES:")
+        lines.append("-" * 50)
+        for entry in results["stale_entries"]:
+            lines.append(f"  [{entry['reason']}] {entry['source_file']}")
+            lines.append(f"    {entry['fact']}")
+            if entry.get("stored_hash") and entry.get("current_hash"):
+                lines.append(f"    stored:  {entry['stored_hash']}")
+                lines.append(f"    current: {entry['current_hash']}")
+            lines.append("")
+
+    if results["total"] > 0:
+        staleness_pct = results["stale"] / results["total"] * 100
+        lines.append(f"Staleness rate: {staleness_pct:.1f}%")
+    else:
+        lines.append("No entries to check.")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Check knowledge store for stale entries")
+    parser.add_argument("--index", default="knowledge/index.json", help="Path to index.json")
+    parser.add_argument("--repo", help="Repository root (default: auto-detect from index path)")
+    parser.add_argument("--fix", action="store_true", help="Add missing hashes to index")
+    parser.add_argument("--json", action="store_true", help="Output JSON instead of report")
+    args = parser.parse_args()
+
+    if args.fix:
+        result = add_hashes_to_index(args.index, args.repo)
+        if args.json:
+            print(json.dumps(result, indent=2))
+        else:
+            print(f"Updated {result['updated']} entries with source hashes.")
+            print(f"Skipped {result['skipped']} (already had hashes or no source file).")
+    else:
+        results = check_staleness(args.index, args.repo)
+        if "error" in results:
+            print(f"Error: {results['error']}", file=sys.stderr)
+            sys.exit(1)
+
+        if args.json:
+            print(json.dumps(results, indent=2))
+        else:
+            print(report_staleness(results))
+
+
+if __name__ == "__main__":
+    main()

scripts/sampler.py

@@ -1,353 +0,0 @@
-#!/usr/bin/env python3
-"""
-sampler.py — Score and rank sessions by harvest value.
-
-With 20k+ sessions on disk, we can't harvest all at once. This script
-scores each session by how likely it is to contain valuable knowledge,
-so the harvester processes the best ones first.
-
-Scoring strategy:
-  - Recency: last 7d=3pts, last 30d=2pts, older=1pt
-  - Length: >50 messages=3pts, >20=2pts, <20=1pt
-  - Repo uniqueness: first session for a repo=5pts, otherwise=1pt
-  - Outcome: failure=3pts (most to learn), success=2pts, unknown=1pt
-  - Tool calls: >10 tool invocations=2pts (complex sessions)
-
-Usage:
-    python3 sampler.py --count 100                          # Top 100 sessions
-    python3 sampler.py --repo the-nexus --count 20          # Top 20 for a repo
-    python3 sampler.py --since 2026-04-01                   # All sessions since date
-    python3 sampler.py --count 50 --min-score 8             # Only high-value sessions
-    python3 sampler.py --count 100 --output sample.json     # Save to file
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-from typing import Optional
-
-
-# --- Fast session scanning (no full parse) ---
-
-def scan_session_fast(path: str) -> dict:
-    """Extract scoring metadata from a session without parsing the full JSONL.
-    
-    Reads only: first line, last ~20 lines, and line count. This processes
-    20k sessions in seconds instead of minutes.
-    """
-    meta = {
-        'path': path,
-        'message_count': 0,
-        'has_tool_calls': False,
-        'tool_call_count': 0,
-        'first_timestamp': '',
-        'last_timestamp': '',
-        'is_failure': False,
-        'repos_mentioned': set(),
-        'first_role': '',
-        'last_content_preview': '',
-    }
-    
-    try:
-        file_size = os.path.getsize(path)
-        if file_size == 0:
-            return meta
-        
-        with open(path, 'r', encoding='utf-8', errors='replace') as f:
-            # Read first line for timestamp + role
-            first_line = f.readline().strip()
-            if first_line:
-                try:
-                    first_msg = json.loads(first_line)
-                    meta['first_timestamp'] = first_msg.get('timestamp', '')
-                    meta['first_role'] = first_msg.get('role', '')
-                except json.JSONDecodeError:
-                    pass
-            
-            # Fast line count + collect tail lines
-            # For the tail, seek to near end of file
-            tail_lines = []
-            line_count = 1  # already read first
-            
-            if file_size > 8192:
-                # Seek to last 8KB for tail sampling
-                f.seek(max(0, file_size - 8192))
-                f.readline()  # skip partial line
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        tail_lines.append(line)
-                    line_count += 1
-                # We lost the exact count for big files — estimate from file size
-                # Average JSONL line is ~500 bytes
-                if line_count < 100:
-                    line_count = max(line_count, file_size // 500)
-            else:
-                # Small file — read all
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        tail_lines.append(line)
-                    line_count += 1
-            
-            meta['message_count'] = line_count
-            
-            # Parse tail lines for outcome, tool calls, repos
-            for line in tail_lines[-30:]:  # last 30 non-empty lines
-                try:
-                    msg = json.loads(line)
-                    
-                    # Track last timestamp
-                    ts = msg.get('timestamp', '')
-                    if ts:
-                        meta['last_timestamp'] = ts
-                    
-                    # Count tool calls
-                    if msg.get('tool_calls'):
-                        meta['has_tool_calls'] = True
-                        meta['tool_call_count'] += len(msg['tool_calls'])
-                    
-                    # Detect failure signals in content
-                    content = ''
-                    if isinstance(msg.get('content'), str):
-                        content = msg['content'].lower()
-                    elif isinstance(msg.get('content'), list):
-                        for part in msg['content']:
-                            if isinstance(part, dict) and part.get('type') == 'text':
-                                content += part.get('text', '').lower()
-                    
-                    if content:
-                        meta['last_content_preview'] = content[:200]
-                        failure_signals = ['error', 'failed', 'cannot', 'unable', 
-                                          'exception', 'traceback', 'rejected', 'denied']
-                        if any(sig in content for sig in failure_signals):
-                            meta['is_failure'] = True
-                    
-                    # Extract repo references from tool call arguments
-                    if msg.get('tool_calls'):
-                        for tc in msg['tool_calls']:
-                            args = tc.get('function', {}).get('arguments', '')
-                            if isinstance(args, str):
-                                # Look for repo patterns
-                                for pattern in ['Timmy_Foundation/', 'Rockachopa/', 'compounding-intelligence', 'the-nexus', 'timmy-home', 'hermes-agent', 'the-beacon', 'the-door']:
-                                    if pattern in args:
-                                        repo = pattern.rstrip('/')
-                                        meta['repos_mentioned'].add(repo)
-                
-                except json.JSONDecodeError:
-                    continue
-    
-    except (IOError, OSError):
-        pass
-    
-    meta['repos_mentioned'] = list(meta['repos_mentioned'])
-    return meta
-
-
-# --- Filename timestamp parsing ---
-
-def parse_session_timestamp(filename: str) -> Optional[datetime]:
-    """Parse timestamp from session filename.
-    
-    Common formats:
-      session_20260413_123456_hash.jsonl
-      20260413_123456_hash.jsonl
-    """
-    stem = Path(filename).stem
-    parts = stem.split('_')
-    
-    # Try session_YYYYMMDD_HHMMSS format
-    for i, part in enumerate(parts):
-        if len(part) == 8 and part.isdigit():
-            date_part = part
-            time_part = parts[i + 1] if i + 1 < len(parts) and len(parts[i + 1]) == 6 else '000000'
-            try:
-                return datetime.strptime(f"{date_part}_{time_part}", '%Y%m%d_%H%M%S').replace(tzinfo=timezone.utc)
-            except ValueError:
-                continue
-    
-    # Fallback: use file modification time
-    return None
-
-
-# --- Scoring ---
-
-def score_session(meta: dict, now: datetime, seen_repos: set) -> tuple[int, dict]:
-    """Score a session for harvest value. Returns (score, breakdown)."""
-    score = 0
-    breakdown = {}
-    
-    # 1. Recency
-    ts = parse_session_timestamp(os.path.basename(meta['path']))
-    if ts is None:
-        # Fallback to mtime
-        try:
-            ts = datetime.fromtimestamp(os.path.getmtime(meta['path']), tz=timezone.utc)
-        except OSError:
-            ts = now - timedelta(days=365)
-    
-    age_days = (now - ts).days
-    if age_days <= 7:
-        recency = 3
-    elif age_days <= 30:
-        recency = 2
-    else:
-        recency = 1
-    score += recency
-    breakdown['recency'] = recency
-    
-    # 2. Length
-    count = meta['message_count']
-    if count > 50:
-        length = 3
-    elif count > 20:
-        length = 2
-    else:
-        length = 1
-    score += length
-    breakdown['length'] = length
-    
-    # 3. Repo uniqueness (first session mentioning a repo gets bonus)
-    repo_score = 0
-    for repo in meta.get('repos_mentioned', []):
-        if repo not in seen_repos:
-            seen_repos.add(repo)
-            repo_score = max(repo_score, 5)
-        else:
-            repo_score = max(repo_score, 1)
-    score += repo_score
-    breakdown['repo_unique'] = repo_score
-    
-    # 4. Outcome
-    if meta.get('is_failure'):
-        outcome = 3
-    elif meta.get('last_content_preview', '').strip():
-        outcome = 2  # has some content = likely completed
-    else:
-        outcome = 1
-    score += outcome
-    breakdown['outcome'] = outcome
-    
-    # 5. Tool calls
-    if meta.get('tool_call_count', 0) > 10:
-        tool = 2
-    else:
-        tool = 0
-    score += tool
-    breakdown['tool_calls'] = tool
-    
-    return score, breakdown
-
-
-# --- Main ---
-
-def main():
-    parser = argparse.ArgumentParser(description="Score and rank sessions for harvesting")
-    parser.add_argument('--sessions-dir', default=os.path.expanduser('~/.hermes/sessions'),
-                        help='Directory containing session files')
-    parser.add_argument('--count', type=int, default=100, help='Number of top sessions to return')
-    parser.add_argument('--repo', default='', help='Filter to sessions mentioning this repo')
-    parser.add_argument('--since', default='', help='Only score sessions after this date (YYYY-MM-DD)')
-    parser.add_argument('--min-score', type=int, default=0, help='Minimum score threshold')
-    parser.add_argument('--output', default='', help='Output file (JSON). Default: stdout')
-    parser.add_argument('--format', choices=['json', 'paths', 'table'], default='table',
-                        help='Output format: json (full), paths (one per line), table (human)')
-    parser.add_argument('--top-percent', type=float, default=0, help='Return top N%% instead of --count')
-    
-    args = parser.parse_args()
-    
-    sessions_dir = Path(args.sessions_dir)
-    if not sessions_dir.is_dir():
-        print(f"ERROR: Sessions directory not found: {sessions_dir}", file=sys.stderr)
-        sys.exit(1)
-    
-    # Find all JSONL files
-    print(f"Scanning {sessions_dir}...", file=sys.stderr)
-    t0 = time.time()
-    
-    session_files = list(sessions_dir.glob('*.jsonl'))
-    total = len(session_files)
-    print(f"Found {total} session files", file=sys.stderr)
-    
-    # Parse since date
-    since_dt = None
-    if args.since:
-        since_dt = datetime.strptime(args.since, '%Y-%m-%d').replace(tzinfo=timezone.utc)
-    
-    # Score all sessions
-    now = datetime.now(timezone.utc)
-    seen_repos = set()  # Track repos for uniqueness scoring
-    scored = []
-    
-    for i, sf in enumerate(session_files):
-        # Date filter (fast path: check filename first)
-        if since_dt:
-            ts = parse_session_timestamp(sf.name)
-            if ts and ts < since_dt:
-                continue
-        
-        meta = scan_session_fast(str(sf))
-        
-        # Repo filter
-        if args.repo:
-            repos = meta.get('repos_mentioned', [])
-            if args.repo.lower() not in [r.lower() for r in repos]:
-                # Also check filename
-                if args.repo.lower() not in sf.name.lower():
-                    continue
-        
-        score, breakdown = score_session(meta, now, seen_repos)
-        
-        if score >= args.min_score:
-            scored.append({
-                'path': str(sf),
-                'filename': sf.name,
-                'score': score,
-                'breakdown': breakdown,
-                'message_count': meta['message_count'],
-                'repos': meta['repos_mentioned'],
-                'is_failure': meta['is_failure'],
-            })
-        
-        if (i + 1) % 5000 == 0:
-            elapsed = time.time() - t0
-            print(f"  Scanned {i + 1}/{total} ({elapsed:.1f}s)", file=sys.stderr)
-    
-    elapsed = time.time() - t0
-    print(f"Scored {len(scored)} sessions in {elapsed:.1f}s", file=sys.stderr)
-    
-    # Sort by score descending
-    scored.sort(key=lambda x: x['score'], reverse=True)
-    
-    # Apply count or percent
-    if args.top_percent > 0:
-        count = max(1, int(len(scored) * args.top_percent / 100))
-    else:
-        count = args.count
-    scored = scored[:count]
-    
-    # Output
-    if args.output:
-        with open(args.output, 'w', encoding='utf-8') as f:
-            json.dump(scored, f, indent=2)
-        print(f"Wrote {len(scored)} sessions to {args.output}", file=sys.stderr)
-    elif args.format == 'json':
-        json.dump(scored, sys.stdout, indent=2)
-    elif args.format == 'paths':
-        for s in scored:
-            print(s['path'])
-    else:  # table
-        print(f"{'SCORE':>5}  {'MSGS':>5}  {'REPOS':<25}  {'FILE'}")
-        print(f"{'-'*5}  {'-'*5}  {'-'*25}  {'-'*40}")
-        for s in scored:
-            repos = ', '.join(s['repos'][:2]) if s['repos'] else '-'
-            fail = ' FAIL' if s['is_failure'] else ''
-            print(f"{s['score']:>5}  {s['message_count']:>5}  {repos:<25}  {s['filename'][:40]}{fail}")
-
-
-if __name__ == '__main__':
-    main()