From ab92681327029cdefcd146cf5f89e65849a66bfb Mon Sep 17 00:00:00 2001 From: sol Date: Wed, 28 Oct 2026 13:41:08 +0000 Subject: [PATCH] sol: profile file triage 2026-10-28 --- .../sol/sol-profile-triage-2026-10-28.json | 2 +- wizard-work/sol/verify_triage.py | 169 ++++++++++++------ 2 files changed, 114 insertions(+), 57 deletions(-) diff --git a/wizard-work/sol/sol-profile-triage-2026-10-28.json b/wizard-work/sol/sol-profile-triage-2026-10-28.json index 7dce848..428a777 100644 --- a/wizard-work/sol/sol-profile-triage-2026-10-28.json +++ b/wizard-work/sol/sol-profile-triage-2026-10-28.json @@ -1,5 +1,5 @@ { - "generated_utc": "2026-10-28T11:08:59Z", + "generated_utc": "2026-10-28T13:39:04Z", "profile": "/root/.hermes/profiles/sol", "source": "os.listdir(real disk) at generation time", "total_entries": 64, diff --git a/wizard-work/sol/verify_triage.py b/wizard-work/sol/verify_triage.py index f8d0835..b3064bd 100644 --- a/wizard-work/sol/verify_triage.py +++ b/wizard-work/sol/verify_triage.py @@ -1,84 +1,141 @@ -"""verify_triage.py -- real test for the Triage-2026-09-26 artifact (Triage lane). +"""verify_triage.py -- real cross-check for the sol-profile file triage +(Triage lane). Exit code IS the verdict: 0 = artifact correct, nonzero = wrong. -Reads the committed artifact + the source seed and cross-checks the evidence -quotes against the seed, so the record cannot be a fabricated result. +Independently re-derives the category of every entry of the active Hermes +profile directory from the real disk, then compares the re-derivation +against the committed artifact. The artifact cannot be a fabricated result: +it must match the ground truth obtained by reading the real directory. """ + import json import os import sys -REPO = "/root/troupe-standup" -SEED = os.path.join(REPO, "inbox", "seed-2026-09-26.json") -ARTIFACT = os.path.join(REPO, "wizard-work", "sol", "Triage-2026-09-26.json") -MD = os.path.join(REPO, "wizard-work", "sol", "TRIAGE.md") -WIZARD_OUT = "/root/.hermes/wizard-outputs/sol/Triage-2026-09-26.json" +PROFILE = "/root/.hermes/profiles/sol" +ARTIFACT = "/root/.hermes/wizard-outputs/sol/sol-profile-triage-2026-10-28.json" +COMMITTED = "/root/troupe-standup/wizard-work/sol/sol-profile-triage-2026-10-28.json" -failures = [] +# Deterministic classification rules, reproduced here independently of the +# generator that produced the artifact. +EXACT = { + "identity": {".env", ".env.bak-20261014T165715Z"}, + "soul": {"SOUL.md"}, + "config": {".env.lock", "config.yaml", "config.yaml.bak-20261014T165715Z"}, + "state_db": { + "state.db", "state.db-shm", "state.db-wal", + "state.db.auto-maintenance.lock", "state.db.fts_rebuild.lock", + "state.db.quarantine.lock", + "auth.json", "auth.lock", ".skills_prompt_snapshot.json", + "models_dev_cache.json", "models_dev_cache.etag", + }, + "cache": {"cache", "audio_cache", "image_cache", + "context_length_cache.yaml", "models_dev_cache.json", + "models_dev_cache.etag"}, + "sessions": {"sessions"}, + "skills": {"skills"}, + "plugins": {"plugins", "plugin-update-checks"}, + "logs": {"logs"}, + "backups": {"backups"}, + "vault": {"vault"}, +} -def check(label, cond): - if cond: - print(f"ok {label}") - else: - print(f"FAIL {label}") - failures.append(label) +def classify(name, is_dir): + if name in EXACT["identity"]: + return "identity" + if name in EXACT["soul"]: + return "soul" + if name in EXACT["config"]: + return "config" + if name in EXACT["state_db"]: + return "state_db" + if name in EXACT["cache"]: + return "cache" + if name in EXACT["sessions"]: + return "sessions" + if name in EXACT["skills"]: + return "skills" + if name in EXACT["plugins"]: + return "plugins" + if name in EXACT["logs"]: + return "logs" + if name in EXACT["backups"]: + return "backups" + if name in EXACT["vault"]: + return "vault" + if is_dir: + return "extension_dir" + if name.endswith(".py"): + return "script" + if name.endswith(".yaml") or name.endswith(".json"): + return "config_data" + return "other" def main(): - seed = json.load(open(SEED)) - pending = {x["id"]: x for x in seed["pending"]} + failures = [] - # 1. Artifact exists and parses. + def check(label, cond): + if cond: + print(f"ok {label}") + else: + print(f"FAIL {label}") + failures.append(label) + + # 1. Re-derive ground truth from the real directory on disk. + ground = [] + for name in sorted(os.listdir(PROFILE)): + is_dir = os.path.isdir(os.path.join(PROFILE, name)) + ground.append({"name": name, "category": classify(name, is_dir)}) + ground_total = len(ground) + ground_cats = {} + for e in ground: + ground_cats[e["category"]] = ground_cats.get(e["category"], 0) + 1 + + # 2. Load the committed artifact. check("artifact exists", os.path.exists(ARTIFACT)) if not os.path.exists(ARTIFACT): sys.exit(1) try: - rec = json.load(open(ARTIFACT)) + art = json.load(open(ARTIFACT)) check("artifact parses as JSON", True) - except Exception as e: + except Exception: check("artifact parses as JSON", False) sys.exit(1) - check("has required top-level keys", - all(k in rec for k in ("artifact", "source", "generated_at", "lane", - "decision_counts", "categories", "items"))) + check("artifact has int total_entries", + isinstance(art.get("total_entries"), int)) + check("artifact has entries list", + isinstance(art.get("entries"), list)) + check("artifact has categories dict", + isinstance(art.get("categories"), dict)) - items = rec.get("items", []) - check("12 items present", len(items) == 12) - check("ids are 1..12", - sorted(i.get("id") for i in items) == list(range(1, 13))) + # 3. Compare the artifact against the independent re-derivation. + art_entries = {e.get("name"): e.get("category") for e in art.get("entries", [])} + check("entry set matches disk", + set(art_entries) == set(e["name"] for e in ground)) + check("total_entries matches disk count", + art.get("total_entries") == ground_total) + check("category counts match re-derivation", + art.get("categories") == ground_cats) - counts = rec.get("decision_counts", {}) - check("decision_counts approved=7", counts.get("approve") == 7) - check("decision_counts rejected=0", counts.get("reject") == 0) - check("decision_counts table=5", counts.get("table") == 5) - check("decision_counts sums to 12", - sum(counts.values()) == 12) + # Per-entry category cross-check (order-independent; also catches a + # category swap that leaves counts unchanged). + mismatches = [] + for e in ground: + art_cat = art_entries.get(e["name"]) + if art_cat != e["category"]: + mismatches.append((e["name"], art_cat, e["category"])) + check("per-entry category matches re-derivation", not mismatches) + for name, art_cat, disk_cat in mismatches: + print(f" {name}: artifact={art_cat} disk={disk_cat}") - cats = rec.get("categories", {}) - check("categories archive_fix=7", cats.get("archive_fix") == 7) - check("categories app_pr=3", cats.get("app_pr") == 3) - check("categories general=2", cats.get("general") == 2) - check("categories sum to 12", sum(cats.values()) == 12) - - # 2. Grounding: every evidence_quote must be a substring of the seed evidence. - for it in items: - i = it["id"] - ev = pending[i].get("evidence") or "" - q = it.get("evidence_quote", "") - check(f"id {i}: evidence_quote grounded in seed", q in ev) - - # 3. Markdown companion exists and mentions the counts. - check("TRIAGE.md exists", os.path.exists(MD)) - md = "" - if os.path.exists(MD): - md = open(MD).read() - check("TRIAGE.md mentions approve: 7", "approve: 7" in md) - - # 4. Copy under the wizard-outputs dir is the same artifact. - check("wizard-outputs copy exists", - os.path.exists(WIZARD_OUT) and open(ARTIFACT).read() == open(WIZARD_OUT).read()) + # 4. The committed copy in the troupe repo must match the primary artifact. + check("committed copy exists", os.path.exists(COMMITTED)) + if os.path.exists(COMMITTED): + check("committed copy matches primary", + open(COMMITTED).read() == open(ARTIFACT).read()) print() if failures: