Compare commits
1 Commits
step35/134
...
step35/91-
| Author | SHA1 | Date | |
|---|---|---|---|
|
b1a728f5f4 |
@@ -1,258 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""GitHub Trending Scanner — Scan trending repos in AI/ML.
|
||||
|
||||
Extracts: repo description, stars, key features (topics, inferred highlights).
|
||||
Filters by language and/or topic. Outputs dated JSON for daily scan pipeline.
|
||||
|
||||
Usage:
|
||||
python3 github_trending_scanner.py --language python --topic ai --output metrics/trending
|
||||
python3 github_trending_scanner.py --topic machine-learning --limit 50
|
||||
python3 github_trending_scanner.py --language rust --topic artificial-intelligence
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
|
||||
GITHUB_API_BASE = os.environ.get("GITHUB_API_BASE", "https://api.github.com")
|
||||
DEFAULT_OUTPUT_DIR = os.environ.get("TRENDING_OUTPUT_DIR", "metrics/trending")
|
||||
DEFAULT_LIMIT = int(os.environ.get("TRENDING_LIMIT", "30"))
|
||||
DEFAULT_MIN_STARS = int(os.environ.get("TRENDING_MIN_STARS", "1000"))
|
||||
|
||||
|
||||
def fetch_trending_repos(
|
||||
language: Optional[str] = None,
|
||||
topic: Optional[str] = None,
|
||||
min_stars: int = DEFAULT_MIN_STARS,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
) -> List[Dict]:
|
||||
"""Fetch trending-like repositories from GitHub using the search API.
|
||||
|
||||
GitHub's public search API is unauthenticated-rate-limited (60 req/hr).
|
||||
This function retries on rate-limit backoff and falls back gracefully.
|
||||
"""
|
||||
# Build search query: stars threshold + optional language/topic filters
|
||||
query = f"stars:>{min_stars}"
|
||||
if language:
|
||||
query += f" language:{language}"
|
||||
if topic:
|
||||
query += f" topic:{topic}"
|
||||
|
||||
# Sort by stars descending as a proxy for trending/popular
|
||||
params = {
|
||||
"q": query,
|
||||
"sort": "stars",
|
||||
"order": "desc",
|
||||
"per_page": min(limit, 100), # GitHub max per_page is 100
|
||||
}
|
||||
url = f"{GITHUB_API_BASE}/search/repositories?{urllib.parse.urlencode(params)}"
|
||||
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "Sovereign-Trending-Scanner/1.0",
|
||||
}
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"GitHub API returned {resp.status}")
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
return data.get("items", [])[:limit]
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 403:
|
||||
# Check for rate limit message
|
||||
body = e.read().decode("utf-8", errors="replace").lower()
|
||||
if "rate limit" in body or "api rate limit exceeded" in body:
|
||||
reset_ts = int(e.headers.get("X-RateLimit-Reset", 0))
|
||||
wait_seconds = max(5, reset_ts - int(time.time()) + 5)
|
||||
print(f"Rate limit exceeded — waiting {wait_seconds}s (attempt {attempt+1}/3)...", file=sys.stderr)
|
||||
time.sleep(wait_seconds)
|
||||
continue
|
||||
print(f"ERROR: GitHub API request failed: {e} — {e.read().decode('utf-8', errors='replace')[:200]}", file=sys.stderr)
|
||||
return []
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
backoff = 2 ** attempt
|
||||
print(f"WARNING: Fetch attempt {attempt+1} failed: {e} — retrying in {backoff}s", file=sys.stderr)
|
||||
time.sleep(backoff)
|
||||
continue
|
||||
print(f"ERROR: All fetch attempts failed: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def extract_repo_features(repo_data: Dict) -> Dict:
|
||||
"""Extract structured fields for a trending repo."""
|
||||
description = (repo_data.get("description") or "").strip()
|
||||
topics = repo_data.get("topics", [])
|
||||
|
||||
# Infer key features from description and topics
|
||||
features = infer_features(description, topics)
|
||||
|
||||
return {
|
||||
"name": repo_data.get("full_name", ""),
|
||||
"description": description,
|
||||
"stars": repo_data.get("stargazers_count", 0),
|
||||
"forks": repo_data.get("forks_count", 0),
|
||||
"open_issues": repo_data.get("open_issues_count", 0),
|
||||
"language": repo_data.get("language", ""),
|
||||
"topics": topics,
|
||||
"url": repo_data.get("html_url", ""),
|
||||
"created_at": repo_data.get("created_at", ""),
|
||||
"updated_at": repo_data.get("updated_at", ""),
|
||||
"key_features": features,
|
||||
"scanned_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def infer_features(description: str, topics: List[str]) -> List[str]:
|
||||
"""Infer notable capabilities/features from repo metadata.
|
||||
|
||||
Looks for AI/ML-relevant capabilities in topics and description.
|
||||
"""
|
||||
features = []
|
||||
text = (description + " " + " ".join(topics)).lower()
|
||||
|
||||
# Domain capabilities (keys normalized to lowercase for consistency)
|
||||
capability_keywords = {
|
||||
"fine-tuning": ["fine-tun", "finetun"],
|
||||
"agent framework": ["agent"],
|
||||
"local/offline": ["local", "on-device", "offline"],
|
||||
"quantized models": ["quantized", "quantization", "gguf", "gptq"],
|
||||
"vision": ["vision", "multimodal", "image", "visual"],
|
||||
"speech/audio": ["speech", "audio", "whisper", "tts"],
|
||||
"retrieval/rag": ["rag", "retrieval", "embedding", "vector"],
|
||||
"training": ["train", "training", "sft", "dpo"],
|
||||
"gui/playground": ["gui", "playground", "webui", "interface"],
|
||||
"sota": ["state-of-the-art", "sota", "latest"],
|
||||
}
|
||||
|
||||
for label, keywords in capability_keywords.items():
|
||||
if any(kw in text for kw in keywords):
|
||||
features.append(label)
|
||||
|
||||
# Also include non-generic topics as features
|
||||
generic_topics = {"ai", "ml", "machine-learning", "deep-learning", "llm", "python", "pytorch", "tensorflow"}
|
||||
for topic in topics:
|
||||
if topic.lower() not in generic_topics:
|
||||
features.append(topic)
|
||||
|
||||
# Deduplicate while preserving order, return up to 10
|
||||
seen = set()
|
||||
unique = []
|
||||
for f in features:
|
||||
key = f.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(f)
|
||||
return unique[:10]
|
||||
|
||||
|
||||
def save_trending(repos: List[Dict], output_dir: str = "metrics/trending") -> str:
|
||||
"""Save trending results to a dated JSON file.
|
||||
|
||||
Returns the path of the written file.
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
filename = output_path / f"github-trending-{date_str}.json"
|
||||
|
||||
output_data = {
|
||||
"scanned_at": datetime.now(timezone.utc).isoformat(),
|
||||
"count": len(repos),
|
||||
"repos": repos,
|
||||
}
|
||||
|
||||
with open(filename, "w") as f:
|
||||
json.dump(output_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return str(filename)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scan GitHub trending repositories in AI/ML"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
help="Filter by programming language (e.g., python, rust, go)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--topic",
|
||||
help="Filter by GitHub topic (e.g., ai, machine-learning, llm)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--since",
|
||||
default="daily",
|
||||
choices=["daily", "weekly", "monthly"],
|
||||
help="Trending period (daily/weekly/monthly) — informational only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="metrics/trending",
|
||||
help="Output directory for results (default: metrics/trending)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=DEFAULT_LIMIT,
|
||||
help=f"Maximum repos to fetch (default: {DEFAULT_LIMIT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-stars",
|
||||
type=int,
|
||||
default=DEFAULT_MIN_STARS,
|
||||
help=f"Minimum star count for relevance (default: {DEFAULT_MIN_STARS})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(
|
||||
f"Fetching trending repos "
|
||||
f"(language={args.language or 'any'}, topic={args.topic or 'any'}, period={args.since})..."
|
||||
)
|
||||
|
||||
repos_raw = fetch_trending_repos(
|
||||
language=args.language,
|
||||
topic=args.topic,
|
||||
min_stars=args.min_stars,
|
||||
limit=args.limit,
|
||||
)
|
||||
|
||||
if not repos_raw:
|
||||
print("WARNING: No repos fetched — check network or rate limits", file=sys.stderr)
|
||||
|
||||
repos = [extract_repo_features(r) for r in repos_raw]
|
||||
|
||||
output_file = save_trending(repos, args.output)
|
||||
print(f"Saved {len(repos)} trending repos to {output_file}")
|
||||
|
||||
# Brief human-readable summary
|
||||
if repos:
|
||||
print("\nTop repos:")
|
||||
for repo in repos[:5]:
|
||||
features_preview = ", ".join(repo["key_features"][:3])
|
||||
print(f" ★ {repo['stars']:>7} {repo['name']}")
|
||||
if repo["description"]:
|
||||
desc = repo["description"][:80]
|
||||
print(f" {desc}{'...' if len(repo['description']) > 80 else ''}")
|
||||
if features_preview:
|
||||
print(f" Features: {features_preview}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -22,114 +22,95 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from session_reader import extract_conversation, read_session
|
||||
|
||||
|
||||
def compute_hash(text: str) -> str:
|
||||
"""Content hash for deduplication."""
|
||||
return hashlib.sha256(text.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def extract_pairs_from_session(session_data: dict, min_ratio: float = 1.5,
|
||||
def extract_pairs_from_conversation(conversation: list, session_id: str, model: str,
|
||||
min_ratio: float = 1.5,
|
||||
min_response_words: int = 20) -> list:
|
||||
"""Extract terse→rich pairs from a single session object."""
|
||||
"""Extract terse→rich pairs from a normalized conversation."""
|
||||
pairs = []
|
||||
conversations = session_data.get("conversations", [])
|
||||
session_id = session_data.get("id", "unknown")
|
||||
model = session_data.get("model", "unknown")
|
||||
|
||||
seen_hashes = set()
|
||||
|
||||
for i, msg in enumerate(conversations):
|
||||
# Look for assistant/gpt responses
|
||||
if msg.get("from") not in ("gpt", "assistant"):
|
||||
for i, msg in enumerate(conversation):
|
||||
# Look for assistant responses
|
||||
if msg.get('role') != 'assistant':
|
||||
continue
|
||||
|
||||
response_text = msg.get("value", "")
|
||||
response_text = msg.get('content', '')
|
||||
if not response_text or len(response_text.split()) < min_response_words:
|
||||
continue
|
||||
|
||||
# Find the preceding human message
|
||||
# Find the preceding user message
|
||||
prompt_text = ""
|
||||
for j in range(i - 1, -1, -1):
|
||||
if conversations[j].get("from") == "human":
|
||||
prompt_text = conversations[j].get("value", "")
|
||||
if conversation[j].get('role') == 'user':
|
||||
prompt_text = conversation[j].get('content', '')
|
||||
break
|
||||
|
||||
if not prompt_text:
|
||||
continue
|
||||
|
||||
# Filter: skip tool results, system messages embedded as human
|
||||
if prompt_text.startswith("{") and "output" in prompt_text[:100]:
|
||||
continue # likely a tool result
|
||||
if prompt_text.startswith("# SOUL.md") or prompt_text.startswith("You are"):
|
||||
continue # system prompt leak
|
||||
if prompt_text.startswith('{') and 'output' in prompt_text[:100]:
|
||||
continue
|
||||
if prompt_text.startswith('# SOUL.md') or prompt_text.startswith('You are'):
|
||||
continue
|
||||
|
||||
# Quality filters
|
||||
prompt_words = len(prompt_text.split())
|
||||
response_words = len(response_text.split())
|
||||
|
||||
# Must have meaningful length ratio
|
||||
if prompt_words == 0 or response_words == 0:
|
||||
continue
|
||||
ratio = response_words / prompt_words
|
||||
if ratio < min_ratio:
|
||||
continue
|
||||
|
||||
# Skip responses that are mostly code
|
||||
code_blocks = response_text.count("```")
|
||||
if code_blocks >= 4 and len(response_text.replace("```", "").strip()) < 50:
|
||||
code_blocks = response_text.count('```')
|
||||
if code_blocks >= 4 and len(response_text.replace('```', '').strip()) < 50:
|
||||
continue
|
||||
|
||||
# Skip responses with tool call artifacts
|
||||
if "tool_call" in response_text[:100] or "function_call" in response_text[:100]:
|
||||
if 'tool_call' in response_text[:100] or 'function_call' in response_text[:100]:
|
||||
continue
|
||||
|
||||
# Deduplicate by content hash
|
||||
content_hash = compute_hash(prompt_text + response_text[:200])
|
||||
if content_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(content_hash)
|
||||
|
||||
# Clean up response: remove markdown headers if too many
|
||||
clean_response = response_text
|
||||
|
||||
pairs.append({
|
||||
"terse": prompt_text.strip(),
|
||||
"rich": clean_response.strip(),
|
||||
"source": session_id,
|
||||
"model": model,
|
||||
"prompt_words": prompt_words,
|
||||
"response_words": response_words,
|
||||
"ratio": round(ratio, 2),
|
||||
'terse': prompt_text.strip(),
|
||||
'rich': clean_response.strip(),
|
||||
'source': session_id,
|
||||
'model': model,
|
||||
'prompt_words': prompt_words,
|
||||
'response_words': response_words,
|
||||
'ratio': round(ratio, 2),
|
||||
})
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def extract_from_jsonl_file(filepath: str, **kwargs) -> list:
|
||||
"""Extract pairs from a session JSONL file."""
|
||||
pairs = []
|
||||
path = Path(filepath)
|
||||
|
||||
if not path.exists():
|
||||
print(f"Warning: {filepath} not found", file=sys.stderr)
|
||||
return pairs
|
||||
|
||||
content = path.read_text()
|
||||
lines = content.strip().split("\n")
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
session = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
session_pairs = extract_pairs_from_session(session, **kwargs)
|
||||
pairs.extend(session_pairs)
|
||||
|
||||
return pairs
|
||||
def extract_from_jsonl_file(path: str, **kwargs) -> list:
|
||||
"""Read a session file and extract training pairs using normalized conversation."""
|
||||
session_messages = read_session(path)
|
||||
if not session_messages:
|
||||
return []
|
||||
conversation = extract_conversation(session_messages)
|
||||
# Derive session_id and model from first real message metadata
|
||||
first_msg = next((m for m in session_messages if m.get('role') or m.get('from')), {})
|
||||
session_id = first_msg.get('meta_session_id', Path(path).name)
|
||||
model = first_msg.get('model', 'unknown')
|
||||
return extract_pairs_from_conversation(conversation, session_id, model, **kwargs)
|
||||
|
||||
|
||||
def deduplicate_pairs(pairs: list) -> list:
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for github_trending_scanner.py — pure function validation.
|
||||
|
||||
Tests the feature inference, extraction, and output formatting logic
|
||||
without relying on external GitHub API calls.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
# Add scripts dir to path for import
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from github_trending_scanner import (
|
||||
extract_repo_features,
|
||||
infer_features,
|
||||
save_trending,
|
||||
)
|
||||
|
||||
|
||||
def test_infer_features_from_description():
|
||||
"""Feature inference extracts capabilities from description text."""
|
||||
desc = "A local, quantized LLM framework for fine-tuning and agent-based RAG with vision."
|
||||
topics = ["ai", "llm"]
|
||||
features = infer_features(desc, topics)
|
||||
|
||||
# Should include relevant capabilities (case-insensitive comparison)
|
||||
expected_lower = {"fine-tuning", "local/offline", "quantized models", "agent framework", "vision", "retrieval/rag"}
|
||||
actual_lower = set(f.lower() for f in features)
|
||||
assert expected_lower.issubset(actual_lower), f"Missing features. Expected subset of {expected_lower}, got {actual_lower}"
|
||||
print("PASS: infer_features_from_description")
|
||||
|
||||
|
||||
def test_infer_features_from_topics_only():
|
||||
"""Topics alone can drive feature detection."""
|
||||
desc = ""
|
||||
topics = ["computer-vision", "speech", "pytorch"]
|
||||
features = infer_features(desc, topics)
|
||||
|
||||
# Non-generic topics should appear as features (topics preserved as-is)
|
||||
assert "computer-vision" in features, f"Expected 'computer-vision' in {features}"
|
||||
assert "speech" in features, f"Expected 'speech' in {features}"
|
||||
# Generic topics (pytorch) may be filtered
|
||||
print(f"PASS: infer_features_from_topics_only → {features}")
|
||||
|
||||
|
||||
def test_extract_repo_features_produces_valid_structure():
|
||||
"""extract_repo_features returns all required fields."""
|
||||
mock_repo = {
|
||||
"full_name": "example/repo",
|
||||
"description": "An example repository",
|
||||
"stargazers_count": 1234,
|
||||
"forks_count": 56,
|
||||
"open_issues_count": 7,
|
||||
"language": "Python",
|
||||
"topics": ["ai", "llm"],
|
||||
"html_url": "https://github.com/example/repo",
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"updated_at": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
result = extract_repo_features(mock_repo)
|
||||
|
||||
assert result["name"] == "example/repo"
|
||||
assert result["description"] == "An example repository"
|
||||
assert result["stars"] == 1234
|
||||
assert isinstance(result["key_features"], list)
|
||||
assert "scanned_at" in result
|
||||
assert result["url"] == "https://github.com/example/repo"
|
||||
print("PASS: extract_repo_features_structure")
|
||||
|
||||
|
||||
def test_save_trending_creates_dated_json():
|
||||
"""save_trending writes a valid JSON file with the expected schema."""
|
||||
repos = [
|
||||
{
|
||||
"name": "test/repo",
|
||||
"description": "Test repository",
|
||||
"stars": 999,
|
||||
"language": "Python",
|
||||
"topics": ["test"],
|
||||
"key_features": ["testing"],
|
||||
"scanned_at": "2026-04-26T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output_file = save_trending(repos, output_dir=tmp)
|
||||
|
||||
path = Path(output_file)
|
||||
assert path.exists(), f"Output file not created: {output_file}"
|
||||
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert "scanned_at" in data
|
||||
assert data["count"] == 1
|
||||
assert isinstance(data["repos"], list)
|
||||
assert data["repos"][0]["name"] == "test/repo"
|
||||
print(f"PASS: save_trending → {output_file}")
|
||||
|
||||
|
||||
def test_save_trending_respects_output_dir_creation():
|
||||
"""Output directory is created if it doesn't exist."""
|
||||
repos = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
nested = Path(tmp) / "nested" / "trending"
|
||||
assert not nested.exists()
|
||||
|
||||
output_file = save_trending(repos, output_dir=str(nested))
|
||||
assert nested.exists()
|
||||
assert Path(output_file).exists()
|
||||
print("PASS: output_dir_creation")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_infer_features_from_description()
|
||||
test_infer_features_from_topics_only()
|
||||
test_extract_repo_features_produces_valid_structure()
|
||||
test_save_trending_creates_dated_json()
|
||||
test_save_trending_respects_output_dir_creation()
|
||||
print("\nAll github_trending_scanner tests passed.")
|
||||
118
tests/test_session_pair_harvester.py
Normal file
118
tests/test_session_pair_harvester.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Tests for session_pair_harvester — training pair extraction from sessions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
from session_pair_harvester import (
|
||||
extract_pairs_from_conversation,
|
||||
extract_from_jsonl_file,
|
||||
deduplicate_pairs,
|
||||
compute_hash,
|
||||
)
|
||||
|
||||
|
||||
class TestSessionPairHarvester(unittest.TestCase):
|
||||
def test_compute_hash_consistent(self):
|
||||
h1 = compute_hash("hello world")
|
||||
h2 = compute_hash("hello world")
|
||||
self.assertEqual(h1, h2)
|
||||
self.assertEqual(len(h1), 16)
|
||||
|
||||
def test_extract_simple_qa_pair(self):
|
||||
"""A simple user→assistant exchange produces one pair."""
|
||||
conversation = [
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
{"role": "assistant", "content": "The capital of France is Paris. It is a major European city renowned for its art, fashion, gastronomy, cultural heritage, and historical significance. The city attracts millions of tourists annually."},
|
||||
]
|
||||
pairs = extract_pairs_from_conversation(conversation, "test_session", "test-model")
|
||||
self.assertEqual(len(pairs), 1)
|
||||
self.assertEqual(pairs[0]["terse"], "What is the capital of France?")
|
||||
self.assertIn("Paris", pairs[0]["rich"])
|
||||
self.assertEqual(pairs[0]["source"], "test_session")
|
||||
|
||||
def test_min_ratio_filter(self):
|
||||
"""Very short responses are filtered out."""
|
||||
conversation = [
|
||||
{"role": "user", "content": "Yes"},
|
||||
{"role": "assistant", "content": "No."},
|
||||
]
|
||||
# Default min_ratio = 1.5, min_words = 20 for response
|
||||
pairs = extract_pairs_from_conversation(conversation, "s", "m", min_response_words=3)
|
||||
self.assertEqual(len(pairs), 0)
|
||||
|
||||
def test_min_words_filter(self):
|
||||
"""Assistant responses below min word count are skipped."""
|
||||
conversation = [
|
||||
{"role": "user", "content": "Explain the project architecture in detail"},
|
||||
{"role": "assistant", "content": "OK."},
|
||||
]
|
||||
pairs = extract_pairs_from_conversation(conversation, "s", "m", min_response_words=5)
|
||||
self.assertEqual(len(pairs), 0)
|
||||
|
||||
def test_skip_non_assistant_messages(self):
|
||||
"""System and tool messages are ignored."""
|
||||
conversation = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there! How can I help you today?"},
|
||||
]
|
||||
pairs = extract_pairs_from_conversation(conversation, "s", "m", min_response_words=3)
|
||||
self.assertEqual(len(pairs), 1)
|
||||
self.assertEqual(pairs[0]["terse"], "Hello")
|
||||
|
||||
def test_multiple_pairs_from_one_session(self):
|
||||
"""A conversation with several Q&A turns yields multiple pairs."""
|
||||
conversation = [
|
||||
{"role": "user", "content": "First question?"},
|
||||
{"role": "assistant", "content": "Here is a detailed and comprehensive answer that thoroughly explores multiple aspects of the subject. It provides background context and practical implications for the reader."},
|
||||
{"role": "user", "content": "Second?"},
|
||||
{"role": "assistant", "content": "Another comprehensive response with detailed examples. This includes practical code blocks and thorough explanations to ensure deep understanding of the topic at hand."},
|
||||
]
|
||||
pairs = extract_pairs_from_conversation(conversation, "s", "m", min_ratio=1.0)
|
||||
self.assertEqual(len(pairs), 2)
|
||||
|
||||
def test_deduplication_removes_duplicates(self):
|
||||
"""Identical pairs across sessions are deduplicated."""
|
||||
pairs = [
|
||||
{"terse": "q1", "rich": "a1", "source": "s1", "model": "m"},
|
||||
{"terse": "q1", "rich": "a1", "source": "s2", "model": "m"},
|
||||
{"terse": "q2", "rich": "a2", "source": "s1", "model": "m"},
|
||||
]
|
||||
unique = deduplicate_pairs(pairs)
|
||||
self.assertEqual(len(unique), 2)
|
||||
sources = {p["source"] for p in unique}
|
||||
# First unique pair can be from either s1 or s2
|
||||
self.assertIn("s1", sources)
|
||||
|
||||
def test_integration_with_test_sessions(self):
|
||||
"""Harvester finds pairs in real test session files."""
|
||||
repo_root = Path(__file__).parent.parent
|
||||
test_sessions_dir = repo_root / "test_sessions"
|
||||
if not test_sessions_dir.exists():
|
||||
self.skipTest("test_sessions not found")
|
||||
|
||||
pairs = []
|
||||
for jsonl_file in sorted(test_sessions_dir.glob("*.jsonl")):
|
||||
pairs.extend(extract_from_jsonl_file(str(jsonl_file)))
|
||||
|
||||
self.assertGreater(len(pairs), 0, "Should extract at least one pair from test_sessions")
|
||||
for p in pairs:
|
||||
self.assertIn("terse", p)
|
||||
self.assertIn("rich", p)
|
||||
self.assertIn("source", p)
|
||||
self.assertIn("model", p)
|
||||
# Verify content exists
|
||||
self.assertGreater(len(p["terse"]), 0)
|
||||
self.assertGreater(len(p["rich"]), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user