150 lines
5.0 KiB
Python
150 lines
5.0 KiB
Python
"""verify_triage.py -- real cross-check for the sol-profile file triage
|
|
(Triage lane).
|
|
|
|
Exit code IS the verdict: 0 = artifact correct, nonzero = wrong.
|
|
Independently re-derives the category of every entry of the active Hermes
|
|
profile directory from the real disk, then compares the re-derivation
|
|
against the committed artifact. The artifact cannot be a fabricated result:
|
|
it must match the ground truth obtained by reading the real directory.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
PROFILE = "/root/.hermes/profiles/sol"
|
|
ARTIFACT = "/root/.hermes/wizard-outputs/sol/sol-profile-triage-2026-10-28.json"
|
|
COMMITTED = "/root/troupe-standup/wizard-work/sol/sol-profile-triage-2026-10-28.json"
|
|
|
|
# Deterministic classification rules, reproduced here independently of the
|
|
# generator that produced the artifact.
|
|
EXACT = {
|
|
"identity": {".env", ".env.bak-20261014T165715Z"},
|
|
"soul": {"SOUL.md"},
|
|
"config": {".env.lock", "config.yaml", "config.yaml.bak-20261014T165715Z"},
|
|
"state_db": {
|
|
"state.db", "state.db-shm", "state.db-wal",
|
|
"state.db.auto-maintenance.lock", "state.db.fts_rebuild.lock",
|
|
"state.db.quarantine.lock",
|
|
"auth.json", "auth.lock", ".skills_prompt_snapshot.json",
|
|
"models_dev_cache.json", "models_dev_cache.etag",
|
|
},
|
|
"cache": {"cache", "audio_cache", "image_cache",
|
|
"context_length_cache.yaml", "models_dev_cache.json",
|
|
"models_dev_cache.etag"},
|
|
"sessions": {"sessions"},
|
|
"skills": {"skills"},
|
|
"plugins": {"plugins", "plugin-update-checks"},
|
|
"logs": {"logs"},
|
|
"backups": {"backups"},
|
|
"vault": {"vault"},
|
|
}
|
|
|
|
|
|
def classify(name, is_dir):
|
|
if name in EXACT["identity"]:
|
|
return "identity"
|
|
if name in EXACT["soul"]:
|
|
return "soul"
|
|
if name in EXACT["config"]:
|
|
return "config"
|
|
if name in EXACT["state_db"]:
|
|
return "state_db"
|
|
if name in EXACT["cache"]:
|
|
return "cache"
|
|
if name in EXACT["sessions"]:
|
|
return "sessions"
|
|
if name in EXACT["skills"]:
|
|
return "skills"
|
|
if name in EXACT["plugins"]:
|
|
return "plugins"
|
|
if name in EXACT["logs"]:
|
|
return "logs"
|
|
if name in EXACT["backups"]:
|
|
return "backups"
|
|
if name in EXACT["vault"]:
|
|
return "vault"
|
|
if is_dir:
|
|
return "extension_dir"
|
|
if name.endswith(".py"):
|
|
return "script"
|
|
if name.endswith(".yaml") or name.endswith(".json"):
|
|
return "config_data"
|
|
return "other"
|
|
|
|
|
|
def main():
|
|
failures = []
|
|
|
|
def check(label, cond):
|
|
if cond:
|
|
print(f"ok {label}")
|
|
else:
|
|
print(f"FAIL {label}")
|
|
failures.append(label)
|
|
|
|
# 1. Re-derive ground truth from the real directory on disk.
|
|
ground = []
|
|
for name in sorted(os.listdir(PROFILE)):
|
|
is_dir = os.path.isdir(os.path.join(PROFILE, name))
|
|
ground.append({"name": name, "category": classify(name, is_dir)})
|
|
ground_total = len(ground)
|
|
ground_cats = {}
|
|
for e in ground:
|
|
ground_cats[e["category"]] = ground_cats.get(e["category"], 0) + 1
|
|
|
|
# 2. Load the committed artifact.
|
|
check("artifact exists", os.path.exists(ARTIFACT))
|
|
if not os.path.exists(ARTIFACT):
|
|
sys.exit(1)
|
|
try:
|
|
art = json.load(open(ARTIFACT))
|
|
check("artifact parses as JSON", True)
|
|
except Exception:
|
|
check("artifact parses as JSON", False)
|
|
sys.exit(1)
|
|
|
|
check("artifact has int total_entries",
|
|
isinstance(art.get("total_entries"), int))
|
|
check("artifact has entries list",
|
|
isinstance(art.get("entries"), list))
|
|
check("artifact has categories dict",
|
|
isinstance(art.get("categories"), dict))
|
|
|
|
# 3. Compare the artifact against the independent re-derivation.
|
|
art_entries = {e.get("name"): e.get("category") for e in art.get("entries", [])}
|
|
check("entry set matches disk",
|
|
set(art_entries) == set(e["name"] for e in ground))
|
|
check("total_entries matches disk count",
|
|
art.get("total_entries") == ground_total)
|
|
check("category counts match re-derivation",
|
|
art.get("categories") == ground_cats)
|
|
|
|
# Per-entry category cross-check (order-independent; also catches a
|
|
# category swap that leaves counts unchanged).
|
|
mismatches = []
|
|
for e in ground:
|
|
art_cat = art_entries.get(e["name"])
|
|
if art_cat != e["category"]:
|
|
mismatches.append((e["name"], art_cat, e["category"]))
|
|
check("per-entry category matches re-derivation", not mismatches)
|
|
for name, art_cat, disk_cat in mismatches:
|
|
print(f" {name}: artifact={art_cat} disk={disk_cat}")
|
|
|
|
# 4. The committed copy in the troupe repo must match the primary artifact.
|
|
check("committed copy exists", os.path.exists(COMMITTED))
|
|
if os.path.exists(COMMITTED):
|
|
check("committed copy matches primary",
|
|
open(COMMITTED).read() == open(ARTIFACT).read())
|
|
|
|
print()
|
|
if failures:
|
|
print(f"VERDICT: FAIL ({len(failures)} failed)")
|
|
sys.exit(1)
|
|
print("VERDICT: PASS")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|