Compare commits
1 Commits
fix/687-tr
...
fix/issue-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa3d22d89c |
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Full Nostr agent-to-agent communication demo - FINAL WORKING
|
||||
"""
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Soul Eval Gate — The Conscience of the Training Pipeline
|
||||
|
||||
|
||||
104
cron/audit-report.json
Normal file
104
cron/audit-report.json
Normal file
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"audit_time": "2026-04-15T01:13:31.126215+00:00",
|
||||
"total_jobs": 7,
|
||||
"summary": {
|
||||
"healthy": 7,
|
||||
"transient_errors": 0,
|
||||
"systemic_failures": 0
|
||||
},
|
||||
"systemic_jobs": [],
|
||||
"transient_jobs": [],
|
||||
"all_jobs": [
|
||||
{
|
||||
"id": "9e0624269ba7",
|
||||
"name": "Triage Heartbeat",
|
||||
"schedule": "every 15m",
|
||||
"state": "paused",
|
||||
"enabled": false,
|
||||
"last_status": "ok",
|
||||
"last_error": null,
|
||||
"last_run_at": "2026-03-24T15:33:57.749458-04:00",
|
||||
"category": "healthy",
|
||||
"reason": "Dashboard repo frozen - loops redirected to the-nexus",
|
||||
"action": "none \u2014 paused intentionally"
|
||||
},
|
||||
{
|
||||
"id": "e29eda4a8548",
|
||||
"name": "PR Review Sweep",
|
||||
"schedule": "every 30m",
|
||||
"state": "paused",
|
||||
"enabled": false,
|
||||
"last_status": "ok",
|
||||
"last_error": null,
|
||||
"last_run_at": "2026-03-24T15:21:42.995715-04:00",
|
||||
"category": "healthy",
|
||||
"reason": "Dashboard repo frozen - loops redirected to the-nexus",
|
||||
"action": "none \u2014 paused intentionally"
|
||||
},
|
||||
{
|
||||
"id": "a77a87392582",
|
||||
"name": "Health Monitor",
|
||||
"schedule": "every 5m",
|
||||
"state": "scheduled",
|
||||
"enabled": true,
|
||||
"last_status": "ok",
|
||||
"last_error": null,
|
||||
"last_run_at": "2026-03-24T15:34:39.045945-04:00",
|
||||
"category": "healthy",
|
||||
"reason": "Last run succeeded",
|
||||
"action": ""
|
||||
},
|
||||
{
|
||||
"id": "36fb2f630a17",
|
||||
"name": "Hermes Philosophy Loop",
|
||||
"schedule": "every 1440m",
|
||||
"state": "unknown",
|
||||
"enabled": false,
|
||||
"last_status": null,
|
||||
"last_error": null,
|
||||
"last_run_at": null,
|
||||
"category": "healthy",
|
||||
"reason": "Never run, no errors",
|
||||
"action": ""
|
||||
},
|
||||
{
|
||||
"id": "muda-audit-weekly",
|
||||
"name": "Muda Audit",
|
||||
"schedule": "0 21 * * 0",
|
||||
"state": "scheduled",
|
||||
"enabled": true,
|
||||
"last_status": null,
|
||||
"last_error": null,
|
||||
"last_run_at": null,
|
||||
"category": "healthy",
|
||||
"reason": "Never run, no errors",
|
||||
"action": ""
|
||||
},
|
||||
{
|
||||
"id": "kaizen-retro-349",
|
||||
"name": "Kaizen Retro",
|
||||
"schedule": "daily at 07:30",
|
||||
"state": "scheduled",
|
||||
"enabled": true,
|
||||
"last_status": null,
|
||||
"last_error": null,
|
||||
"last_run_at": null,
|
||||
"category": "healthy",
|
||||
"reason": "Never run, no errors",
|
||||
"action": ""
|
||||
},
|
||||
{
|
||||
"id": "overnight-rd-nightly",
|
||||
"name": "Overnight R&D Loop",
|
||||
"schedule": "Nightly at 10 PM EDT",
|
||||
"state": "scheduled",
|
||||
"enabled": true,
|
||||
"last_status": null,
|
||||
"last_error": null,
|
||||
"last_run_at": null,
|
||||
"category": "healthy",
|
||||
"reason": "Never run, no errors",
|
||||
"action": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
from hermes_tools import browser_navigate, browser_vision
|
||||
|
||||
|
||||
333
scripts/cron-audit-662.py
Normal file
333
scripts/cron-audit-662.py
Normal file
@@ -0,0 +1,333 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cron Fleet Audit Script — #662
|
||||
|
||||
Reads hermes cron job state, categorizes all jobs into:
|
||||
- healthy: last_status=ok or never-run-and-enabled
|
||||
- transient: recent errors (likely network/timeout)
|
||||
- systemic: repeated errors over 48+ hours
|
||||
|
||||
Outputs a JSON report and optionally:
|
||||
--disable Disable systemic jobs erroring 48+ hours
|
||||
--issues File Gitea issues for systemic failures
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
|
||||
# --- Config ---
|
||||
ERROR_THRESHOLD_HOURS = 48
|
||||
CRON_STATE_PATHS = [
|
||||
Path.home() / ".hermes" / "cron" / "jobs.json",
|
||||
Path.home() / ".hermes" / "cron" / "state.json",
|
||||
Path("/root/.hermes/cron/jobs.json"),
|
||||
Path("/root/.hermes/cron/state.json"),
|
||||
]
|
||||
|
||||
def load_cron_state() -> List[Dict[str, Any]]:
|
||||
"""Load cron job state from known locations."""
|
||||
for path in CRON_STATE_PATHS:
|
||||
if path.exists():
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict) and "jobs" in data:
|
||||
return data["jobs"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except (json.JSONDecodeError, IOError):
|
||||
continue
|
||||
|
||||
# Fallback: try hermes cron list CLI
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
["hermes", "cron", "list", "--json"],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
if result.returncode == 0:
|
||||
data = json.loads(result.stdout)
|
||||
if isinstance(data, dict) and "jobs" in data:
|
||||
return data["jobs"]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def parse_timestamp(ts: str) -> datetime:
|
||||
"""Parse ISO timestamp, handle various formats."""
|
||||
if not ts:
|
||||
return None
|
||||
# Normalize timezone
|
||||
ts = ts.replace("+00:00", "+00:00")
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def categorize_job(job: Dict[str, Any], now: datetime) -> Dict[str, Any]:
|
||||
"""Categorize a single job."""
|
||||
status = job.get("last_status", "")
|
||||
last_error = job.get("last_error", "")
|
||||
last_run = parse_timestamp(job.get("last_run_at"))
|
||||
enabled = job.get("enabled", False)
|
||||
state = job.get("state", "unknown")
|
||||
name = job.get("name", job.get("id", "unknown"))
|
||||
|
||||
entry = {
|
||||
"id": job.get("id", ""),
|
||||
"name": name,
|
||||
"schedule": job.get("schedule_display", str(job.get("schedule", ""))),
|
||||
"state": state,
|
||||
"enabled": enabled,
|
||||
"last_status": status,
|
||||
"last_error": last_error,
|
||||
"last_run_at": job.get("last_run_at"),
|
||||
"category": "healthy",
|
||||
"reason": "",
|
||||
"action": "",
|
||||
}
|
||||
|
||||
# Never run / no error
|
||||
if status is None and not last_error:
|
||||
entry["category"] = "healthy"
|
||||
entry["reason"] = "Never run, no errors"
|
||||
return entry
|
||||
|
||||
# Explicitly paused with reason
|
||||
if state == "paused":
|
||||
entry["category"] = "healthy"
|
||||
entry["reason"] = job.get("paused_reason", "Manually paused")
|
||||
entry["action"] = "none — paused intentionally"
|
||||
return entry
|
||||
|
||||
# Completed jobs
|
||||
if state == "completed":
|
||||
entry["category"] = "healthy"
|
||||
entry["reason"] = "Completed (one-shot)"
|
||||
return entry
|
||||
|
||||
# Error status
|
||||
if status == "error" and last_error:
|
||||
age_hours = None
|
||||
if last_run:
|
||||
age_hours = (now - last_run).total_seconds() / 3600
|
||||
|
||||
if age_hours is not None and age_hours >= ERROR_THRESHOLD_HOURS:
|
||||
entry["category"] = "systemic"
|
||||
entry["reason"] = f"Erroring for {age_hours:.1f}h (>{ERROR_THRESHOLD_HOURS}h threshold)"
|
||||
entry["action"] = "disable"
|
||||
else:
|
||||
entry["category"] = "transient"
|
||||
age_str = f"{age_hours:.1f}h ago" if age_hours is not None else "unknown age"
|
||||
entry["reason"] = f"Recent error ({age_str}), may be transient"
|
||||
entry["action"] = "monitor"
|
||||
return entry
|
||||
|
||||
# OK status
|
||||
if status == "ok":
|
||||
entry["category"] = "healthy"
|
||||
entry["reason"] = "Last run succeeded"
|
||||
return entry
|
||||
|
||||
# Scheduled but never errored
|
||||
if state == "scheduled" and enabled:
|
||||
entry["category"] = "healthy"
|
||||
entry["reason"] = "Scheduled and running"
|
||||
return entry
|
||||
|
||||
# Unknown state
|
||||
entry["category"] = "transient"
|
||||
entry["reason"] = f"Unknown state: {state}, status: {status}"
|
||||
entry["action"] = "investigate"
|
||||
return entry
|
||||
|
||||
|
||||
def audit_jobs(jobs: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Run full audit on job list."""
|
||||
now = datetime.now(timezone.utc)
|
||||
categorized = [categorize_job(j, now) for j in jobs]
|
||||
|
||||
healthy = [c for c in categorized if c["category"] == "healthy"]
|
||||
transient = [c for c in categorized if c["category"] == "transient"]
|
||||
systemic = [c for c in categorized if c["category"] == "systemic"]
|
||||
|
||||
report = {
|
||||
"audit_time": now.isoformat(),
|
||||
"total_jobs": len(jobs),
|
||||
"summary": {
|
||||
"healthy": len(healthy),
|
||||
"transient_errors": len(transient),
|
||||
"systemic_failures": len(systemic),
|
||||
},
|
||||
"systemic_jobs": [
|
||||
{
|
||||
"id": j["id"],
|
||||
"name": j["name"],
|
||||
"reason": j["reason"],
|
||||
"last_error": j["last_error"],
|
||||
}
|
||||
for j in systemic
|
||||
],
|
||||
"transient_jobs": [
|
||||
{
|
||||
"id": j["id"],
|
||||
"name": j["name"],
|
||||
"reason": j["reason"],
|
||||
}
|
||||
for j in transient
|
||||
],
|
||||
"all_jobs": categorized,
|
||||
}
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def generate_issue_body(job: Dict[str, Any]) -> str:
|
||||
"""Generate a Gitea issue body for a systemic cron failure."""
|
||||
return f"""## Systemic Cron Failure — Auto-Filed by Audit #662
|
||||
|
||||
**Job:** {job['name']} (`{job['id']}`)
|
||||
**Schedule:** {job['schedule']}
|
||||
**State:** {job['state']}
|
||||
**Last Error:**
|
||||
```
|
||||
{job['last_error'] or 'No error details available'}
|
||||
```
|
||||
|
||||
**Audit Finding:** {job['reason']}
|
||||
|
||||
### Action Required
|
||||
- [ ] Diagnose root cause of repeated failure
|
||||
- [ ] Fix configuration or remove broken job
|
||||
- [ ] Verify job resumes healthy after fix
|
||||
|
||||
*Auto-generated by cron-audit-662.py*
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Cron fleet audit (#662)")
|
||||
parser.add_argument("--jobs-file", help="Path to jobs.json override")
|
||||
parser.add_argument("--disable", action="store_true",
|
||||
help="Disable systemic jobs (requires hermes CLI)")
|
||||
parser.add_argument("--issues", action="store_true",
|
||||
help="File Gitea issues for systemic failures")
|
||||
parser.add_argument("--output", help="Write report to file")
|
||||
parser.add_argument("--json", action="store_true", help="JSON output only")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load jobs
|
||||
jobs = []
|
||||
if args.jobs_file:
|
||||
with open(args.jobs_file) as f:
|
||||
data = json.load(f)
|
||||
jobs = data.get("jobs", data) if isinstance(data, dict) else data
|
||||
else:
|
||||
jobs = load_cron_state()
|
||||
|
||||
if not jobs:
|
||||
print("ERROR: No cron jobs found. Check ~/.hermes/cron/ or run 'hermes cron list'.")
|
||||
sys.exit(1)
|
||||
|
||||
# Run audit
|
||||
report = audit_jobs(jobs)
|
||||
|
||||
# Output
|
||||
if args.json:
|
||||
print(json.dumps(report, indent=2))
|
||||
else:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" CRON FLEET AUDIT — {report['total_jobs']} jobs")
|
||||
print(f"{'='*60}")
|
||||
print(f" Healthy: {report['summary']['healthy']}")
|
||||
print(f" Transient errors: {report['summary']['transient_errors']}")
|
||||
print(f" Systemic failures: {report['summary']['systemic_failures']}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if report["systemic_jobs"]:
|
||||
print(f"\n SYSTEMIC FAILURES (>{ERROR_THRESHOLD_HOURS}h):")
|
||||
for j in report["systemic_jobs"]:
|
||||
print(f" - {j['name']} ({j['id']}): {j['reason']}")
|
||||
if j["last_error"]:
|
||||
print(f" Error: {j['last_error'][:100]}")
|
||||
|
||||
if report["transient_jobs"]:
|
||||
print(f"\n TRANSIENT ERRORS:")
|
||||
for j in report["transient_jobs"]:
|
||||
print(f" - {j['name']} ({j['id']}): {j['reason']}")
|
||||
|
||||
print()
|
||||
|
||||
# Write report file
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
json.dump(report, f, indent=2)
|
||||
print(f"Report written to {args.output}")
|
||||
|
||||
# Disable systemic jobs
|
||||
if args.disable and report["systemic_jobs"]:
|
||||
import subprocess
|
||||
for j in report["systemic_jobs"]:
|
||||
print(f"Disabling: {j['name']} ({j['id']})")
|
||||
try:
|
||||
subprocess.run(
|
||||
["hermes", "cron", "pause", j["id"]],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
print(f" → Disabled")
|
||||
except Exception as e:
|
||||
print(f" → Failed: {e}")
|
||||
|
||||
# File issues for systemic failures
|
||||
if args.issues and report["systemic_jobs"]:
|
||||
gitea_token = os.environ.get("GITEA_TOKEN") or ""
|
||||
if not gitea_token:
|
||||
token_path = Path.home() / ".config" / "gitea" / "token"
|
||||
if token_path.exists():
|
||||
gitea_token = token_path.read_text().strip()
|
||||
|
||||
if not gitea_token:
|
||||
print("ERROR: No Gitea token found. Set GITEA_TOKEN or ~/.config/gitea/token")
|
||||
sys.exit(1)
|
||||
|
||||
import urllib.request
|
||||
base = "https://forge.alexanderwhitestone.com/api/v1"
|
||||
headers = {
|
||||
"Authorization": f"token {gitea_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
for j in report["systemic_jobs"]:
|
||||
title = f"CRON FAIL: {j['name']} — systemic error ({j['id']})"
|
||||
body = generate_issue_body(j)
|
||||
data = json.dumps({"title": title, "body": body}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{base}/repos/Timmy_Foundation/timmy-config/issues",
|
||||
data=data, headers=headers, method="POST"
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req)
|
||||
result = json.loads(resp.read())
|
||||
print(f"Issued #{result['number']}: {title}")
|
||||
except Exception as e:
|
||||
print(f"Failed to file issue for {j['name']}: {e}")
|
||||
|
||||
# Exit code: non-zero if systemic failures found
|
||||
sys.exit(1 if report["systemic_jobs"] else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1
scripts/cron_audit_662.py
Symbolic link
1
scripts/cron_audit_662.py
Symbolic link
@@ -0,0 +1 @@
|
||||
cron-audit-662.py
|
||||
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
from hermes_tools import browser_navigate, browser_vision
|
||||
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
[QUALITY] Training Data Quality Filter
|
||||
Part of the Timmy Foundation tooling.
|
||||
|
||||
Scores and filters JSONL training pairs on specificity, length ratio,
|
||||
and code correctness. Removes low-quality pairs and reports results.
|
||||
|
||||
Usage:
|
||||
python3 scripts/training_quality_filter.py input.jsonl -o filtered.jsonl
|
||||
python3 scripts/training_quality_filter.py input.jsonl --threshold 0.4
|
||||
cat input.jsonl | python3 scripts/training_quality_filter.py -
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import re
|
||||
from typing import Dict, Any, Tuple
|
||||
|
||||
DEFAULT_THRESHOLD = 0.35
|
||||
MIN_TERSE_LEN = 3
|
||||
MIN_RICH_LEN = 10
|
||||
|
||||
|
||||
def score_specificity(terse: str, rich: str) -> float:
|
||||
"""Score how specific the rich response is vs the terse prompt.
|
||||
|
||||
Higher score = more specific, actionable detail in the rich version.
|
||||
"""
|
||||
if not terse or not rich:
|
||||
return 0.0
|
||||
|
||||
# Ratio of unique words (higher = more varied/specific language)
|
||||
rich_words = rich.lower().split()
|
||||
terse_words = terse.lower().split()
|
||||
|
||||
if len(rich_words) < 3:
|
||||
return 0.1
|
||||
|
||||
unique_ratio = len(set(rich_words)) / len(rich_words)
|
||||
|
||||
# Check for concrete details: numbers, file paths, commands, code refs
|
||||
concrete_patterns = [
|
||||
r"\b\d+\b", # numbers
|
||||
r"[/\\]\w+", # file paths
|
||||
r"`[^`]+`", # inline code
|
||||
r"\b(fix|add|remove|update|create|delete|check|run|use)\b", # action verbs
|
||||
]
|
||||
concrete_count = sum(
|
||||
len(re.findall(p, rich, re.IGNORECASE)) for p in concrete_patterns
|
||||
)
|
||||
concrete_score = min(concrete_count / 5.0, 1.0)
|
||||
|
||||
# Length expansion ratio (rich should be meaningfully longer than terse)
|
||||
expansion = len(rich_words) / max(len(terse_words), 1)
|
||||
expansion_score = min(expansion / 5.0, 1.0)
|
||||
|
||||
return round(0.3 * unique_ratio + 0.4 * concrete_score + 0.3 * expansion_score, 3)
|
||||
|
||||
|
||||
def score_length_ratio(terse: str, rich: str) -> float:
|
||||
"""Score the length ratio between terse and rich.
|
||||
|
||||
Too short rich = low quality. Too long = possibly padded.
|
||||
Sweet spot: 3-15x expansion.
|
||||
"""
|
||||
if not terse or not rich:
|
||||
return 0.0
|
||||
|
||||
t_len = len(terse.split())
|
||||
r_len = len(rich.split())
|
||||
|
||||
if t_len < MIN_TERSE_LEN or r_len < MIN_RICH_LEN:
|
||||
return 0.1
|
||||
|
||||
ratio = r_len / max(t_len, 1)
|
||||
|
||||
if ratio < 1.5:
|
||||
return 0.2 # barely expanded
|
||||
elif ratio < 3.0:
|
||||
return 0.5 # some expansion
|
||||
elif ratio <= 15.0:
|
||||
return 1.0 # good expansion
|
||||
elif ratio <= 30.0:
|
||||
return 0.7 # possibly padded
|
||||
else:
|
||||
return 0.4 # very padded
|
||||
|
||||
|
||||
def score_code_correctness(terse: str, rich: str) -> float:
|
||||
"""Score code blocks in the rich response for basic correctness.
|
||||
|
||||
Checks for matching brackets, valid-looking syntax patterns.
|
||||
"""
|
||||
if not rich:
|
||||
return 0.5 # no code = neutral
|
||||
|
||||
code_blocks = re.findall(r"```(?:\w*)\n(.*?)```", rich, re.DOTALL)
|
||||
if not code_blocks:
|
||||
return 0.5 # no code blocks = neutral
|
||||
|
||||
scores = []
|
||||
for block in code_blocks:
|
||||
block_score = 1.0
|
||||
|
||||
# Check bracket balance
|
||||
for open_c, close_c in [("(", ")"), ("[", "]"), ("{", "}")]:
|
||||
if block.count(open_c) != block.count(close_c):
|
||||
block_score -= 0.3
|
||||
|
||||
# Check for common syntax errors
|
||||
if re.search(r"def \w+[^:]*\n(?!\s)", block):
|
||||
block_score -= 0.2 # missing colon or body
|
||||
|
||||
# Minimum viable code length
|
||||
if len(block.strip()) < 10:
|
||||
block_score -= 0.3
|
||||
|
||||
scores.append(max(block_score, 0.0))
|
||||
|
||||
return round(sum(scores) / len(scores), 3) if scores else 0.5
|
||||
|
||||
|
||||
def score_pair(pair: Dict[str, Any]) -> Tuple[float, Dict[str, float]]:
|
||||
"""Score a single training pair. Returns (total_score, breakdown)."""
|
||||
terse = pair.get("terse", "") or pair.get("prompt", "") or ""
|
||||
rich = pair.get("rich", "") or pair.get("response", "") or ""
|
||||
|
||||
spec = score_specificity(terse, rich)
|
||||
length = score_length_ratio(terse, rich)
|
||||
code = score_code_correctness(terse, rich)
|
||||
|
||||
# Weighted total
|
||||
total = round(0.4 * spec + 0.3 * length + 0.3 * code, 3)
|
||||
|
||||
return total, {"specificity": spec, "length_ratio": length, "code_correctness": code}
|
||||
|
||||
|
||||
def filter_pairs(input_path: str, output_path: str, threshold: float,
|
||||
report: bool = False) -> Dict[str, Any]:
|
||||
"""Filter JSONL training pairs by quality score."""
|
||||
kept = []
|
||||
removed = []
|
||||
errors = 0
|
||||
|
||||
source = sys.stdin if input_path == "-" else open(input_path, "r")
|
||||
|
||||
try:
|
||||
for line_num, line in enumerate(source, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
pair = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
score, breakdown = score_pair(pair)
|
||||
entry = {**pair, "_quality_score": score, "_quality_breakdown": breakdown}
|
||||
|
||||
if score >= threshold:
|
||||
kept.append(entry)
|
||||
else:
|
||||
removed.append(entry)
|
||||
finally:
|
||||
if source is not sys.stdin:
|
||||
source.close()
|
||||
|
||||
# Write filtered output
|
||||
if output_path:
|
||||
out = sys.stdout if output_path == "-" else open(output_path, "w")
|
||||
try:
|
||||
for pair in kept:
|
||||
# Strip internal scoring fields before output
|
||||
clean = {k: v for k, v in pair.items() if not k.startswith("_quality")}
|
||||
out.write(json.dumps(clean, ensure_ascii=False) + "\n")
|
||||
finally:
|
||||
if out is not sys.stdin:
|
||||
out.close()
|
||||
|
||||
result = {
|
||||
"total": len(kept) + len(removed),
|
||||
"kept": len(kept),
|
||||
"filtered_out": len(removed),
|
||||
"errors": errors,
|
||||
"threshold": threshold,
|
||||
"filter_rate": round(len(removed) / max(len(kept) + len(removed), 1) * 100, 1),
|
||||
}
|
||||
|
||||
if report and removed:
|
||||
# Show worst offenders
|
||||
removed_sorted = sorted(removed, key=lambda x: x["_quality_score"])
|
||||
result["worst_5"] = [
|
||||
{
|
||||
"score": e["_quality_score"],
|
||||
"terse": (e.get("terse", "") or e.get("prompt", ""))[:80],
|
||||
"breakdown": e["_quality_breakdown"],
|
||||
}
|
||||
for e in removed_sorted[:5]
|
||||
]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Filter training data pairs by quality")
|
||||
parser.add_argument("input", help="Input JSONL file (use - for stdin)")
|
||||
parser.add_argument("-o", "--output", default="-", help="Output JSONL file (default: stdout)")
|
||||
parser.add_argument("-t", "--threshold", type=float, default=DEFAULT_THRESHOLD,
|
||||
help=f"Quality threshold (0.0-1.0, default: {DEFAULT_THRESHOLD})")
|
||||
parser.add_argument("--report", action="store_true", help="Show quality report")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Score only, dont filter")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.dry_run:
|
||||
# Just score and report, no filtering
|
||||
source = sys.stdin if args.input == "-" else open(args.input, "r")
|
||||
scores = []
|
||||
try:
|
||||
for line in source:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
pair = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
score, breakdown = score_pair(pair)
|
||||
scores.append(score)
|
||||
finally:
|
||||
if source is not sys.stdin:
|
||||
source.close()
|
||||
|
||||
if scores:
|
||||
avg = sum(scores) / len(scores)
|
||||
below = sum(1 for s in scores if s < args.threshold)
|
||||
print(f"Total pairs: {len(scores)}")
|
||||
print(f"Average score: {avg:.3f}")
|
||||
print(f"Below threshold ({args.threshold}): {below} ({below/len(scores)*100:.1f}%)")
|
||||
print(f"Min: {min(scores):.3f} Max: {max(scores):.3f} Median: {sorted(scores)[len(scores)//2]:.3f}")
|
||||
return
|
||||
|
||||
result = filter_pairs(args.input, args.output, args.threshold, report=args.report)
|
||||
|
||||
print(f"Training Data Quality Filter", file=sys.stderr)
|
||||
print(f"{'='*40}", file=sys.stderr)
|
||||
print(f"Total pairs: {result['total']}", file=sys.stderr)
|
||||
print(f"Kept: {result['kept']}", file=sys.stderr)
|
||||
print(f"Filtered out: {result['filtered_out']} ({result['filter_rate']}%)", file=sys.stderr)
|
||||
print(f"Errors: {result['errors']}", file=sys.stderr)
|
||||
print(f"Threshold: {result['threshold']}", file=sys.stderr)
|
||||
|
||||
if args.report and "worst_5" in result:
|
||||
print(f"\nWorst 5 pairs:", file=sys.stderr)
|
||||
for w in result["worst_5"]:
|
||||
terse_preview = w["terse"][:60]
|
||||
print(f" [{w['score']:.3f}] {terse_preview}...", file=sys.stderr)
|
||||
bd = w["breakdown"]
|
||||
print(f" spec={bd['specificity']} length={bd['length_ratio']} code={bd['code_correctness']}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
from hermes_tools import browser_navigate, browser_vision
|
||||
|
||||
|
||||
109
tests/test_cron_audit.py
Normal file
109
tests/test_cron_audit.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Tests for scripts/cron-audit-662.py — cron fleet audit.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
# Add scripts to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
from cron_audit_662 import categorize_job, audit_jobs
|
||||
|
||||
|
||||
class TestCategorizeJob(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.now = datetime(2026, 4, 14, 20, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
def test_healthy_ok(self):
|
||||
job = {"id": "a1", "name": "Test", "last_status": "ok", "enabled": True, "state": "scheduled"}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "healthy")
|
||||
|
||||
def test_healthy_never_run(self):
|
||||
job = {"id": "a2", "name": "Never", "last_status": None, "last_error": None}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "healthy")
|
||||
|
||||
def test_healthy_paused(self):
|
||||
job = {"id": "a3", "name": "Paused", "state": "paused", "paused_reason": "intentional"}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "healthy")
|
||||
|
||||
def test_healthy_completed(self):
|
||||
job = {"id": "a4", "name": "Done", "state": "completed"}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "healthy")
|
||||
|
||||
def test_transient_recent_error(self):
|
||||
recent = (self.now - timedelta(hours=2)).isoformat()
|
||||
job = {
|
||||
"id": "t1", "name": "RecentErr",
|
||||
"last_status": "error",
|
||||
"last_error": "Connection timeout",
|
||||
"last_run_at": recent,
|
||||
"enabled": True,
|
||||
"state": "scheduled",
|
||||
}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "transient")
|
||||
self.assertIn("transient", result["reason"].lower())
|
||||
|
||||
def test_systemic_old_error(self):
|
||||
old = (self.now - timedelta(hours=72)).isoformat()
|
||||
job = {
|
||||
"id": "s1", "name": "OldErr",
|
||||
"last_status": "error",
|
||||
"last_error": "ConfigError: bad config",
|
||||
"last_run_at": old,
|
||||
"enabled": True,
|
||||
"state": "scheduled",
|
||||
}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "systemic")
|
||||
self.assertEqual(result["action"], "disable")
|
||||
|
||||
def test_systemic_boundary(self):
|
||||
"""48.1 hours should be systemic."""
|
||||
boundary = (self.now - timedelta(hours=48, minutes=6)).isoformat()
|
||||
job = {
|
||||
"id": "s2", "name": "Boundary",
|
||||
"last_status": "error",
|
||||
"last_error": "fail",
|
||||
"last_run_at": boundary,
|
||||
"enabled": True,
|
||||
"state": "scheduled",
|
||||
}
|
||||
result = categorize_job(job, self.now)
|
||||
self.assertEqual(result["category"], "systemic")
|
||||
|
||||
|
||||
class TestAuditJobs(unittest.TestCase):
|
||||
def test_empty(self):
|
||||
report = audit_jobs([])
|
||||
self.assertEqual(report["total_jobs"], 0)
|
||||
self.assertEqual(report["summary"]["healthy"], 0)
|
||||
|
||||
def test_mixed_report(self):
|
||||
now = datetime(2026, 4, 14, 20, 0, 0, tzinfo=timezone.utc)
|
||||
old = (now - timedelta(hours=72)).isoformat()
|
||||
recent = (now - timedelta(hours=1)).isoformat()
|
||||
|
||||
jobs = [
|
||||
{"id": "h1", "name": "Healthy", "last_status": "ok", "enabled": True, "state": "scheduled"},
|
||||
{"id": "t1", "name": "Transient", "last_status": "error", "last_error": "timeout", "last_run_at": recent, "enabled": True, "state": "scheduled"},
|
||||
{"id": "s1", "name": "Systemic", "last_status": "error", "last_error": "config bad", "last_run_at": old, "enabled": True, "state": "scheduled"},
|
||||
{"id": "p1", "name": "Paused", "state": "paused", "paused_reason": "frozen"},
|
||||
]
|
||||
report = audit_jobs(jobs)
|
||||
self.assertEqual(report["summary"]["healthy"], 2)
|
||||
self.assertEqual(report["summary"]["transient_errors"], 1)
|
||||
self.assertEqual(report["summary"]["systemic_failures"], 1)
|
||||
self.assertEqual(len(report["systemic_jobs"]), 1)
|
||||
self.assertEqual(report["systemic_jobs"][0]["name"], "Systemic")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user