sol: sol daily triage 2026-09-26 checker + graded artifact
This commit is contained in:
sol committed
1 parent
3b8315f158
commit
8647248687
1 file changed
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""check_sol_triage_2026_09_26.py -- real test for the sol daily-triage build.
|
||||
|
||||
Reads the finished daily-triage artifact (Triage-2026-09-26.json + TRIAGE.md)
|
||||
built by triage_build_2026_09_26.py and grades it. Exit code IS the verdict:
|
||||
|
||||
0 every stated claim holds
|
||||
1 a claim failed
|
||||
2 could not grade (artifact missing / unreadable)
|
||||
|
||||
No fallbacks, no || true, no bypassed exit codes.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
BASE = "/root/.hermes/wizard-outputs/sol"
|
||||
JSON_PATH = os.path.join(BASE, "Triage-2026-09-26.json")
|
||||
MD_PATH = os.path.join(BASE, "TRIAGE.md")
|
||||
|
||||
ALLOWED_CATS = {"archive_fix", "app_pr", "general"}
|
||||
ALLOWED_DECISIONS = {"approve", "reject", "table"}
|
||||
PASSED, FAILED = [], []
|
||||
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
ok = bool(ok)
|
||||
(PASSED if ok else FAILED).append(name)
|
||||
print(("PASS " if ok else "FAIL ") + name + (" -> " + detail if detail else ""))
|
||||
return ok
|
||||
|
||||
|
||||
def unresolved(name, cause):
|
||||
print("UNRESOLVED %s -> %s (COULD NOT GRADE, not a fault)" % (name, cause))
|
||||
|
||||
|
||||
def load_json():
|
||||
if not os.path.isfile(JSON_PATH):
|
||||
unresolved("artifact readable", "Triage-2026-09-26.json missing")
|
||||
return None
|
||||
try:
|
||||
with open(JSON_PATH, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
unresolved("artifact readable", "%s: %s" % (type(e).__name__, e))
|
||||
return None
|
||||
|
||||
|
||||
def load_md():
|
||||
if not os.path.isfile(MD_PATH):
|
||||
unresolved("note readable", "TRIAGE.md missing")
|
||||
return None
|
||||
with open(MD_PATH, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def main():
|
||||
data = load_json()
|
||||
if data is None:
|
||||
print("ARTIFACT UNREADABLE")
|
||||
return 2
|
||||
|
||||
# --- structure ----------------------------------------------------------
|
||||
for key in ("artifact", "source", "generated_at", "lane", "decided_in_seed",
|
||||
"pending_in_seed", "summary", "counts", "categories", "items"):
|
||||
check("top-level key present: " + key, key in data, repr(data.get(key)))
|
||||
|
||||
check("generated_at is ISO-8601 UTC",
|
||||
bool(re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$", data.get("generated_at", ""))),
|
||||
data.get("generated_at"))
|
||||
|
||||
check("lane string",
|
||||
isinstance(data.get("lane"), str) and "Triage" in data["lane"],
|
||||
data.get("lane"))
|
||||
|
||||
items = data.get("items")
|
||||
check("items is a list of 12", isinstance(items, list) and len(items) == 12,
|
||||
"len=%s" % (len(items) if isinstance(items, list) else "n/a"))
|
||||
|
||||
ids = []
|
||||
for it in items or []:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
try:
|
||||
ids.append(int(it["id"]))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
check("ids are 1..12 exactly", sorted(ids) == list(range(1, 13)), str(sorted(ids)))
|
||||
|
||||
# --- per-item completeness ----------------------------------------------
|
||||
for it in items:
|
||||
if not isinstance(it, dict):
|
||||
check("item is dict", False, repr(it))
|
||||
continue
|
||||
n = it.get("id")
|
||||
for field in ("category", "decision", "rule", "summary", "evidence"):
|
||||
check("item %s has %s" % (n, field), field in it and isinstance(it[field], str)
|
||||
and it[field].strip(), ("len=%s" % len(it.get(field, ""))) if field in it else "missing")
|
||||
check("item %s category allowed" % n, it.get("category") in ALLOWED_CATS,
|
||||
it.get("category"))
|
||||
check("item %s decision allowed" % n, it.get("decision") in ALLOWED_DECISIONS,
|
||||
it.get("decision"))
|
||||
check("item %s evidence non-empty" % n,
|
||||
bool(isinstance(it.get("evidence"), str) and it["evidence"].strip()),
|
||||
"evidence len=%s" % len(it.get("evidence", "")))
|
||||
|
||||
# --- aggregate consistency ----------------------------------------------
|
||||
cats, decs = {}, {}
|
||||
for it in items:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
c, d = it.get("category"), it.get("decision")
|
||||
if c:
|
||||
cats[c] = cats.get(c, 0) + 1
|
||||
if d:
|
||||
decs[d] = decs.get(d, 0) + 1
|
||||
|
||||
check("counts sum to 12",
|
||||
sum(data.get("counts", {}).values()) == 12,
|
||||
str(data.get("counts")))
|
||||
check("categories sum to 12", sum(cats.values()) == 12, str(cats))
|
||||
check("decision counts match items",
|
||||
decs.get("approve", 0) == data.get("counts", {}).get("approve")
|
||||
and decs.get("table", 0) == data.get("counts", {}).get("table")
|
||||
and decs.get("reject", 0) == data.get("counts", {}).get("reject"),
|
||||
"items=%s counts=%s" % (decs, data.get("counts")))
|
||||
check("categories match items",
|
||||
cats == data.get("categories"), "items=%s file=%s" % (cats, data.get("categories")))
|
||||
check("approve=7, table=5, reject=0",
|
||||
data.get("counts") == {"approve": 7, "reject": 0, "table": 5},
|
||||
str(data.get("counts")))
|
||||
check("decided_in_seed=1, pending_in_seed=12",
|
||||
data.get("decided_in_seed") == 1 and data.get("pending_in_seed") == 12,
|
||||
"decided=%s pending=%s" % (data.get("decided_in_seed"), data.get("pending_in_seed")))
|
||||
|
||||
check("7 archive_fix items", cats.get("archive_fix") == 7, str(cats.get("archive_fix")))
|
||||
check("3 app_pr items", cats.get("app_pr") == 3, str(cats.get("app_pr")))
|
||||
check("2 general items", cats.get("general") == 2, str(cats.get("general")))
|
||||
|
||||
# --- markdown note consistency ------------------------------------------
|
||||
md = load_md()
|
||||
if md is None:
|
||||
return 2
|
||||
check("note has title", md.startswith("# Triage - 2026-09-26"),
|
||||
md.splitlines()[0] if md else "empty")
|
||||
check("note has decision table", "| # | Category | Decision |" in md, "table header present")
|
||||
for i in range(1, 13):
|
||||
check("note lists id %d" % i, re.search(r"\|\s*%d\s*\|" % i, md) is not None, "row %d" % i)
|
||||
check("note approve=7",
|
||||
re.search(r"^- approve: 7$", md, re.M) is not None, "approve line")
|
||||
check("note reject=0",
|
||||
re.search(r"^- reject: 0$", md, re.M) is not None, "reject line")
|
||||
check("note table=5",
|
||||
re.search(r"^- table \(needs a decision\): 5$", md, re.M) is not None, "table line")
|
||||
check("note category counts",
|
||||
re.search(r"^- archive_fix: 7$", md, re.M) is not None
|
||||
and re.search(r"^- app_pr: 3$", md, re.M) is not None
|
||||
and re.search(r"^- general: 2$", md, re.M) is not None,
|
||||
"3 category lines")
|
||||
|
||||
print("\n%d passed, %d failed" % (len(PASSED), len(FAILED)))
|
||||
if FAILED:
|
||||
print("FAILED: " + ", ".join(FAILED))
|
||||
return 1
|
||||
print("ALL CHECKS GREEN")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user