volt
4740166e9d
The Q119 DONE record shipped at 2615c32 without its artifacts: classify.py and
verify.sh were modified and test_classify_fallback.py untracked in the worktree,
so every path the record names read MISSING at the tree about to be committed
(path-claims leg: 2 new rows). The three files land here byte-for-byte as the
record pins them -- classify.py md5 89a5b9d3476265069a0bf2694af9510e (162 lines),
verify.sh md5 4747faef183017c79620d3daf5d918c3 (105 lines),
test_classify_fallback.py md5 4bc2b53d4bcde155616d4741c708bc4a (159 lines) --
and bash wizard-work/sol/verify.sh reads CHECK OK, Ran 10 tests OK, rc 0 in this
tree. Not claimed: that anything in Q119's grading changed; this is its landing.
106 lines
3.5 KiB
Bash
Executable File
106 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Verify the sol-lane triage classifier end to end.
|
|
#
|
|
# Q119: this script used to regenerate the committed `actual.csv` in place and
|
|
# diff it against `expected.csv`, so every run dirtied the tree it was grading
|
|
# and the "known-good" file was produced by the same rules it was checking.
|
|
# It now writes only into a throwaway directory and grades the JSON contract
|
|
# plus the committed fallback cells. Exit codes follow the repo contract:
|
|
# 0 clean, 1 a graded fault, 2 could not grade.
|
|
set -u
|
|
cd "$(dirname "$(readlink -f "$0")")" || exit 2
|
|
|
|
if ! python3 --version >/dev/null 2>&1; then
|
|
echo "verify: python3 not available -- cannot grade" >&2
|
|
exit 2
|
|
fi
|
|
|
|
SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/sol-classify.XXXXXX")" || {
|
|
echo "verify: cannot create scratch dir -- cannot grade" >&2
|
|
exit 2
|
|
}
|
|
cleanup() { rm -rf "$SCRATCH"; }
|
|
trap cleanup EXIT
|
|
|
|
BEFORE_GIT="$(git status --porcelain -uno 2>/dev/null)"
|
|
IN_WORKTREE=0
|
|
git rev-parse --is-inside-work-tree >/dev/null 2>&1 && IN_WORKTREE=1
|
|
# Checksums of every artifact this directory ships, so the cleanliness leg is
|
|
# not vacuous outside a git worktree (a copy in /tmp is still graded).
|
|
snapshot() {
|
|
md5sum classify.py demands.json expected.csv actual.csv actual.json \
|
|
Triage-2026-10-29.json seed-2026-10-29.json 2>/dev/null
|
|
}
|
|
BEFORE_MD5="$(snapshot)"
|
|
|
|
# 1) the classifier runs on the committed seed and writes outside the tree
|
|
python3 classify.py --seed seed-2026-10-29.json --output "$SCRATCH/out.json"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "FAIL: classification run exited $rc"
|
|
exit 1
|
|
fi
|
|
if [ ! -s "$SCRATCH/out.json" ]; then
|
|
echo "FAIL: classifier wrote no artifact"
|
|
exit 1
|
|
fi
|
|
|
|
# 2) JSON contract: every item is labelled, the aggregate sums to the item count
|
|
python3 - "$SCRATCH/out.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
art = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
cats = art["categories"]
|
|
items = art["items"]
|
|
real = {"archive_fix", "app_pr", "general"}
|
|
allowed = real | {"unclassified"}
|
|
problems = []
|
|
if not items:
|
|
problems.append("artifact has no items")
|
|
if sum(cats.values()) != len(items):
|
|
problems.append("categories sum %d != %d items" % (sum(cats.values()), len(items)))
|
|
missing = allowed - set(cats)
|
|
if missing:
|
|
problems.append("categories missing %s" % sorted(missing))
|
|
for it in items:
|
|
if it["category"] not in allowed:
|
|
problems.append("item %r has category %r" % (it.get("id"), it["category"]))
|
|
if it["decision"] not in {"approve", "table", "reject"}:
|
|
problems.append("item %r has decision %r" % (it.get("id"), it["decision"]))
|
|
ids = [it.get("id") for it in items]
|
|
if len(set(ids)) != len(ids):
|
|
problems.append("duplicate item ids")
|
|
if problems:
|
|
raise SystemExit("FAIL: " + "; ".join(problems))
|
|
print("contract ok: %d items, categories=%s" % (len(items), json.dumps(cats, sort_keys=True)))
|
|
PY
|
|
if [ $? -ne 0 ]; then
|
|
echo "FAIL: JSON contract"
|
|
exit 1
|
|
fi
|
|
|
|
# 3) the committed cells (fallback contract, Q119)
|
|
python3 test_classify_fallback.py || {
|
|
echo "FAIL: fallback cells"
|
|
exit 1
|
|
}
|
|
|
|
# 4) the run must not have touched any artifact it grades
|
|
AFTER_MD5="$(snapshot)"
|
|
if [ "$BEFORE_MD5" != "$AFTER_MD5" ]; then
|
|
echo "FAIL: verify.sh changed the tree it grades" >&2
|
|
printf 'before:\n%s\nafter:\n%s\n' "$BEFORE_MD5" "$AFTER_MD5" >&2
|
|
exit 1
|
|
fi
|
|
if [ "$IN_WORKTREE" -eq 1 ]; then
|
|
AFTER_GIT="$(git status --porcelain -uno 2>/dev/null)"
|
|
if [ "$BEFORE_GIT" != "$AFTER_GIT" ]; then
|
|
echo "FAIL: verify.sh changed the worktree it grades" >&2
|
|
printf '%s\n' "$AFTER_GIT" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "CHECK OK"
|