test: add tests for issue body parser (#90)

scripts/gitea_issue_parser.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+Gitea Issue Body Parser
+
+Extracts structured data from Gitea issue markdown bodies:
+- Title
+- Context section
+- Acceptance criteria (checkboxes)
+- Labels
+- Epic/parent references
+
+Usage:
+  python3 scripts/gitea_issue_parser.py <issue_body.txt
+  python3 scripts/gitea_issue_parser.py --url https://forge.../api/v1/repos/.../issues/123
+  echo "issue body" | python3 scripts/gitea_issue_parser.py --stdin
+
+Output: JSON with {title, context, criteria[], labels[], epic_ref}
+"""
+
+import argparse
+import json
+import re
+import sys
+from typing import Optional
+
+
+def parse_issue_body(body: str, title: str = "", labels: list = None) -> dict:
+    """Parse a Gitea issue body into structured JSON."""
+    result = {
+        "title": title,
+        "context": "",
+        "criteria": [],
+        "labels": labels or [],
+        "epic_ref": None,
+        "sections": {},
+    }
+
+    if not body:
+        return result
+
+    # Extract epic/parent reference from title or body
+    epic_pattern = r"#(\d+)"
+    title_refs = re.findall(epic_pattern, title)
+    body_refs = re.findall(epic_pattern, body[:200])  # Check early body refs
+
+    # Look for "Closes #N" or "Part of #N" or "Epic: #N"
+    close_match = re.search(r"(?:Closes?|Fixes?|Resolves?)\s+#(\d+)", body, re.IGNORECASE)
+    part_match = re.search(r"(?:Part of|Epic|Parent|Blocks?)\s+#(\d+)", body, re.IGNORECASE)
+
+    if close_match:
+        result["epic_ref"] = f"#{close_match.group(1)}"
+    elif part_match:
+        result["epic_ref"] = f"#{part_match.group(1)}"
+    elif title_refs:
+        result["epic_ref"] = f"#{title_refs[0]}"
+    elif body_refs:
+        result["epic_ref"] = f"#{body_refs[0]}"
+
+    # Split into sections by ## headers
+    section_pattern = r"^##\s+(.+)$"
+    lines = body.split("\n")
+    current_section = None
+    current_content = []
+
+    for line in lines:
+        header_match = re.match(section_pattern, line)
+        if header_match:
+            # Save previous section
+            if current_section:
+                result["sections"][current_section] = "\n".join(current_content).strip()
+            current_section = header_match.group(1).strip().lower()
+            current_content = []
+        else:
+            current_content.append(line)
+
+    # Save last section
+    if current_section:
+        result["sections"][current_section] = "\n".join(current_content).strip()
+
+    # Extract context
+    for key in ["context", "background", "description", "problem"]:
+        if key in result["sections"]:
+            result["context"] = result["sections"][key]
+            break
+
+    # Extract acceptance criteria (checkboxes)
+    criteria_section = None
+    for key in ["acceptance criteria", "acceptance_criteria", "criteria", "requirements", "definition of done"]:
+        if key in result["sections"]:
+            criteria_section = result["sections"][key]
+            break
+
+    if criteria_section:
+        checkbox_pattern = r"-\s*\[[ xX]?\]\s*(.+)"
+        for match in re.finditer(checkbox_pattern, criteria_section):
+            result["criteria"].append(match.group(1).strip())
+
+        # Also try plain numbered/bulleted lists if no checkboxes found
+        if not result["criteria"]:
+            list_pattern = r"^\s*(?:\d+\.|-|\*)\s+(.+)"
+            for match in re.finditer(list_pattern, criteria_section, re.MULTILINE):
+                result["criteria"].append(match.group(1).strip())
+
+    # If no sectioned criteria found, scan whole body for checkboxes
+    if not result["criteria"]:
+        for match in re.finditer(r"-\s*\[[ xX]?\]\s*(.+)", body):
+            result["criteria"].append(match.group(1).strip())
+
+    return result
+
+
+def parse_from_url(api_url: str, token: str = None) -> dict:
+    """Parse an issue from a Gitea API URL."""
+    import urllib.request
+
+    headers = {}
+    if token:
+        headers["Authorization"] = f"token {token}"
+
+    req = urllib.request.Request(api_url, headers=headers)
+    resp = json.loads(urllib.request.urlopen(req, timeout=30).read())
+
+    title = resp.get("title", "")
+    body = resp.get("body", "")
+    labels = [l["name"] for l in resp.get("labels", [])]
+
+    return parse_issue_body(body, title, labels)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Parse Gitea issue body into structured JSON")
+    parser.add_argument("input", nargs="?", help="Issue body file (or - for stdin)")
+    parser.add_argument("--url", help="Gitea API URL for the issue")
+    parser.add_argument("--stdin", action="store_true", help="Read from stdin")
+    parser.add_argument("--token", help="Gitea API token (or set GITEA_TOKEN env var)")
+    parser.add_argument("--title", default="", help="Issue title (for epic ref extraction)")
+    parser.add_argument("--labels", nargs="*", default=[], help="Issue labels")
+    parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON output")
+    args = parser.parse_args()
+
+    import os
+    token = args.token or os.environ.get("GITEA_TOKEN")
+
+    if args.url:
+        result = parse_from_url(args.url, token)
+    elif args.stdin or (args.input and args.input == "-"):
+        body = sys.stdin.read()
+        result = parse_issue_body(body, args.title, args.labels)
+    elif args.input:
+        with open(args.input) as f:
+            body = f.read()
+        result = parse_issue_body(body, args.title, args.labels)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    indent = 2 if args.pretty else None
+    print(json.dumps(result, indent=indent))
+
+
+if __name__ == "__main__":
+    main()

scripts/sampler.py

@@ -1,353 +0,0 @@
-#!/usr/bin/env python3
-"""
-sampler.py — Score and rank sessions by harvest value.
-
-With 20k+ sessions on disk, we can't harvest all at once. This script
-scores each session by how likely it is to contain valuable knowledge,
-so the harvester processes the best ones first.
-
-Scoring strategy:
-  - Recency: last 7d=3pts, last 30d=2pts, older=1pt
-  - Length: >50 messages=3pts, >20=2pts, <20=1pt
-  - Repo uniqueness: first session for a repo=5pts, otherwise=1pt
-  - Outcome: failure=3pts (most to learn), success=2pts, unknown=1pt
-  - Tool calls: >10 tool invocations=2pts (complex sessions)
-
-Usage:
-    python3 sampler.py --count 100                          # Top 100 sessions
-    python3 sampler.py --repo the-nexus --count 20          # Top 20 for a repo
-    python3 sampler.py --since 2026-04-01                   # All sessions since date
-    python3 sampler.py --count 50 --min-score 8             # Only high-value sessions
-    python3 sampler.py --count 100 --output sample.json     # Save to file
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-from typing import Optional
-
-
-# --- Fast session scanning (no full parse) ---
-
-def scan_session_fast(path: str) -> dict:
-    """Extract scoring metadata from a session without parsing the full JSONL.
-    
-    Reads only: first line, last ~20 lines, and line count. This processes
-    20k sessions in seconds instead of minutes.
-    """
-    meta = {
-        'path': path,
-        'message_count': 0,
-        'has_tool_calls': False,
-        'tool_call_count': 0,
-        'first_timestamp': '',
-        'last_timestamp': '',
-        'is_failure': False,
-        'repos_mentioned': set(),
-        'first_role': '',
-        'last_content_preview': '',
-    }
-    
-    try:
-        file_size = os.path.getsize(path)
-        if file_size == 0:
-            return meta
-        
-        with open(path, 'r', encoding='utf-8', errors='replace') as f:
-            # Read first line for timestamp + role
-            first_line = f.readline().strip()
-            if first_line:
-                try:
-                    first_msg = json.loads(first_line)
-                    meta['first_timestamp'] = first_msg.get('timestamp', '')
-                    meta['first_role'] = first_msg.get('role', '')
-                except json.JSONDecodeError:
-                    pass
-            
-            # Fast line count + collect tail lines
-            # For the tail, seek to near end of file
-            tail_lines = []
-            line_count = 1  # already read first
-            
-            if file_size > 8192:
-                # Seek to last 8KB for tail sampling
-                f.seek(max(0, file_size - 8192))
-                f.readline()  # skip partial line
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        tail_lines.append(line)
-                    line_count += 1
-                # We lost the exact count for big files — estimate from file size
-                # Average JSONL line is ~500 bytes
-                if line_count < 100:
-                    line_count = max(line_count, file_size // 500)
-            else:
-                # Small file — read all
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        tail_lines.append(line)
-                    line_count += 1
-            
-            meta['message_count'] = line_count
-            
-            # Parse tail lines for outcome, tool calls, repos
-            for line in tail_lines[-30:]:  # last 30 non-empty lines
-                try:
-                    msg = json.loads(line)
-                    
-                    # Track last timestamp
-                    ts = msg.get('timestamp', '')
-                    if ts:
-                        meta['last_timestamp'] = ts
-                    
-                    # Count tool calls
-                    if msg.get('tool_calls'):
-                        meta['has_tool_calls'] = True
-                        meta['tool_call_count'] += len(msg['tool_calls'])
-                    
-                    # Detect failure signals in content
-                    content = ''
-                    if isinstance(msg.get('content'), str):
-                        content = msg['content'].lower()
-                    elif isinstance(msg.get('content'), list):
-                        for part in msg['content']:
-                            if isinstance(part, dict) and part.get('type') == 'text':
-                                content += part.get('text', '').lower()
-                    
-                    if content:
-                        meta['last_content_preview'] = content[:200]
-                        failure_signals = ['error', 'failed', 'cannot', 'unable', 
-                                          'exception', 'traceback', 'rejected', 'denied']
-                        if any(sig in content for sig in failure_signals):
-                            meta['is_failure'] = True
-                    
-                    # Extract repo references from tool call arguments
-                    if msg.get('tool_calls'):
-                        for tc in msg['tool_calls']:
-                            args = tc.get('function', {}).get('arguments', '')
-                            if isinstance(args, str):
-                                # Look for repo patterns
-                                for pattern in ['Timmy_Foundation/', 'Rockachopa/', 'compounding-intelligence', 'the-nexus', 'timmy-home', 'hermes-agent', 'the-beacon', 'the-door']:
-                                    if pattern in args:
-                                        repo = pattern.rstrip('/')
-                                        meta['repos_mentioned'].add(repo)
-                
-                except json.JSONDecodeError:
-                    continue
-    
-    except (IOError, OSError):
-        pass
-    
-    meta['repos_mentioned'] = list(meta['repos_mentioned'])
-    return meta
-
-
-# --- Filename timestamp parsing ---
-
-def parse_session_timestamp(filename: str) -> Optional[datetime]:
-    """Parse timestamp from session filename.
-    
-    Common formats:
-      session_20260413_123456_hash.jsonl
-      20260413_123456_hash.jsonl
-    """
-    stem = Path(filename).stem
-    parts = stem.split('_')
-    
-    # Try session_YYYYMMDD_HHMMSS format
-    for i, part in enumerate(parts):
-        if len(part) == 8 and part.isdigit():
-            date_part = part
-            time_part = parts[i + 1] if i + 1 < len(parts) and len(parts[i + 1]) == 6 else '000000'
-            try:
-                return datetime.strptime(f"{date_part}_{time_part}", '%Y%m%d_%H%M%S').replace(tzinfo=timezone.utc)
-            except ValueError:
-                continue
-    
-    # Fallback: use file modification time
-    return None
-
-
-# --- Scoring ---
-
-def score_session(meta: dict, now: datetime, seen_repos: set) -> tuple[int, dict]:
-    """Score a session for harvest value. Returns (score, breakdown)."""
-    score = 0
-    breakdown = {}
-    
-    # 1. Recency
-    ts = parse_session_timestamp(os.path.basename(meta['path']))
-    if ts is None:
-        # Fallback to mtime
-        try:
-            ts = datetime.fromtimestamp(os.path.getmtime(meta['path']), tz=timezone.utc)
-        except OSError:
-            ts = now - timedelta(days=365)
-    
-    age_days = (now - ts).days
-    if age_days <= 7:
-        recency = 3
-    elif age_days <= 30:
-        recency = 2
-    else:
-        recency = 1
-    score += recency
-    breakdown['recency'] = recency
-    
-    # 2. Length
-    count = meta['message_count']
-    if count > 50:
-        length = 3
-    elif count > 20:
-        length = 2
-    else:
-        length = 1
-    score += length
-    breakdown['length'] = length
-    
-    # 3. Repo uniqueness (first session mentioning a repo gets bonus)
-    repo_score = 0
-    for repo in meta.get('repos_mentioned', []):
-        if repo not in seen_repos:
-            seen_repos.add(repo)
-            repo_score = max(repo_score, 5)
-        else:
-            repo_score = max(repo_score, 1)
-    score += repo_score
-    breakdown['repo_unique'] = repo_score
-    
-    # 4. Outcome
-    if meta.get('is_failure'):
-        outcome = 3
-    elif meta.get('last_content_preview', '').strip():
-        outcome = 2  # has some content = likely completed
-    else:
-        outcome = 1
-    score += outcome
-    breakdown['outcome'] = outcome
-    
-    # 5. Tool calls
-    if meta.get('tool_call_count', 0) > 10:
-        tool = 2
-    else:
-        tool = 0
-    score += tool
-    breakdown['tool_calls'] = tool
-    
-    return score, breakdown
-
-
-# --- Main ---
-
-def main():
-    parser = argparse.ArgumentParser(description="Score and rank sessions for harvesting")
-    parser.add_argument('--sessions-dir', default=os.path.expanduser('~/.hermes/sessions'),
-                        help='Directory containing session files')
-    parser.add_argument('--count', type=int, default=100, help='Number of top sessions to return')
-    parser.add_argument('--repo', default='', help='Filter to sessions mentioning this repo')
-    parser.add_argument('--since', default='', help='Only score sessions after this date (YYYY-MM-DD)')
-    parser.add_argument('--min-score', type=int, default=0, help='Minimum score threshold')
-    parser.add_argument('--output', default='', help='Output file (JSON). Default: stdout')
-    parser.add_argument('--format', choices=['json', 'paths', 'table'], default='table',
-                        help='Output format: json (full), paths (one per line), table (human)')
-    parser.add_argument('--top-percent', type=float, default=0, help='Return top N%% instead of --count')
-    
-    args = parser.parse_args()
-    
-    sessions_dir = Path(args.sessions_dir)
-    if not sessions_dir.is_dir():
-        print(f"ERROR: Sessions directory not found: {sessions_dir}", file=sys.stderr)
-        sys.exit(1)
-    
-    # Find all JSONL files
-    print(f"Scanning {sessions_dir}...", file=sys.stderr)
-    t0 = time.time()
-    
-    session_files = list(sessions_dir.glob('*.jsonl'))
-    total = len(session_files)
-    print(f"Found {total} session files", file=sys.stderr)
-    
-    # Parse since date
-    since_dt = None
-    if args.since:
-        since_dt = datetime.strptime(args.since, '%Y-%m-%d').replace(tzinfo=timezone.utc)
-    
-    # Score all sessions
-    now = datetime.now(timezone.utc)
-    seen_repos = set()  # Track repos for uniqueness scoring
-    scored = []
-    
-    for i, sf in enumerate(session_files):
-        # Date filter (fast path: check filename first)
-        if since_dt:
-            ts = parse_session_timestamp(sf.name)
-            if ts and ts < since_dt:
-                continue
-        
-        meta = scan_session_fast(str(sf))
-        
-        # Repo filter
-        if args.repo:
-            repos = meta.get('repos_mentioned', [])
-            if args.repo.lower() not in [r.lower() for r in repos]:
-                # Also check filename
-                if args.repo.lower() not in sf.name.lower():
-                    continue
-        
-        score, breakdown = score_session(meta, now, seen_repos)
-        
-        if score >= args.min_score:
-            scored.append({
-                'path': str(sf),
-                'filename': sf.name,
-                'score': score,
-                'breakdown': breakdown,
-                'message_count': meta['message_count'],
-                'repos': meta['repos_mentioned'],
-                'is_failure': meta['is_failure'],
-            })
-        
-        if (i + 1) % 5000 == 0:
-            elapsed = time.time() - t0
-            print(f"  Scanned {i + 1}/{total} ({elapsed:.1f}s)", file=sys.stderr)
-    
-    elapsed = time.time() - t0
-    print(f"Scored {len(scored)} sessions in {elapsed:.1f}s", file=sys.stderr)
-    
-    # Sort by score descending
-    scored.sort(key=lambda x: x['score'], reverse=True)
-    
-    # Apply count or percent
-    if args.top_percent > 0:
-        count = max(1, int(len(scored) * args.top_percent / 100))
-    else:
-        count = args.count
-    scored = scored[:count]
-    
-    # Output
-    if args.output:
-        with open(args.output, 'w', encoding='utf-8') as f:
-            json.dump(scored, f, indent=2)
-        print(f"Wrote {len(scored)} sessions to {args.output}", file=sys.stderr)
-    elif args.format == 'json':
-        json.dump(scored, sys.stdout, indent=2)
-    elif args.format == 'paths':
-        for s in scored:
-            print(s['path'])
-    else:  # table
-        print(f"{'SCORE':>5}  {'MSGS':>5}  {'REPOS':<25}  {'FILE'}")
-        print(f"{'-'*5}  {'-'*5}  {'-'*25}  {'-'*40}")
-        for s in scored:
-            repos = ', '.join(s['repos'][:2]) if s['repos'] else '-'
-            fail = ' FAIL' if s['is_failure'] else ''
-            print(f"{s['score']:>5}  {s['message_count']:>5}  {repos:<25}  {s['filename'][:40]}{fail}")
-
-
-if __name__ == '__main__':
-    main()

scripts/test_gitea_issue_parser.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Tests for gitea_issue_parser."""
+
+import json
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+from gitea_issue_parser import parse_issue_body
+
+
+def test_basic_structure():
+    body = """## Context
+This is the background.
+
+## Acceptance Criteria
+- [ ] First criterion
+- [x] Second criterion (already done)
+- [ ] Third criterion
+
+## Labels
+`pipeline`, `extraction`
+"""
+    result = parse_issue_body(body, "Test Issue", ["pipeline", "extraction"])
+    assert result["title"] == "Test Issue"
+    assert "background" in result["context"].lower()
+    assert len(result["criteria"]) == 3
+    assert "First criterion" in result["criteria"]
+    assert result["labels"] == ["pipeline", "extraction"]
+    print("PASS: test_basic_structure")
+
+
+def test_epic_ref():
+    body = "Closes #645\n\nSome description."
+    result = parse_issue_body(body, "feat: thing (#688)")
+    assert result["epic_ref"] == "#645"
+    print("PASS: test_epic_ref")
+
+
+def test_epic_ref_from_title():
+    body = "Some description without close ref."
+    result = parse_issue_body(body, "feat: scene descriptions (#645)")
+    assert result["epic_ref"] == "#645"
+    print("PASS: test_epic_ref_from_title")
+
+
+def test_no_checkboxes():
+    body = """## Requirements
+1. First thing
+2. Second thing
+3. Third thing
+"""
+    result = parse_issue_body(body)
+    assert len(result["criteria"]) == 3
+    print("PASS: test_no_checkboxes")
+
+
+def test_empty_body():
+    result = parse_issue_body("", "Empty Issue")
+    assert result["title"] == "Empty Issue"
+    assert result["criteria"] == []
+    assert result["context"] == ""
+    print("PASS: test_empty_body")
+
+
+def test_real_issue_format():
+    body = """Closes #681
+
+## Changes
+
+Add `#!/usr/bin/env python3` shebang to 6 Python scripts.
+
+## Verification
+
+All 6 files confirmed missing shebangs before fix.
+
+## Impact
+
+Scripts can now be executed directly.
+"""
+    result = parse_issue_body(body, "fix: add python3 shebangs (#685)")
+    assert result["epic_ref"] == "#681"
+    assert "shebang" in result["context"].lower()
+    print("PASS: test_real_issue_format")
+
+
+def test_all_sections_captured():
+    body = """## Context
+Background info.
+
+## Acceptance Criteria
+- [ ] Do thing
+
+## Labels
+`test`
+"""
+    result = parse_issue_body(body)
+    assert "context" in result["sections"]
+    assert "acceptance criteria" in result["sections"]
+    print("PASS: test_all_sections_captured")
+
+
+if __name__ == "__main__":
+    test_basic_structure()
+    test_epic_ref()
+    test_epic_ref_from_title()
+    test_no_checkboxes()
+    test_empty_body()
+    test_real_issue_format()
+    test_all_sections_captured()
+    print("\nAll tests passed.")