93 lines
3.7 KiB
Bash
93 lines
3.7 KiB
Bash
#!/bin/bash
|
|
# CHECK-q143 -- every CHECK entry point in this tree, graded against a baseline.
|
|
# Two legs by default (selftest, paths); `--full` adds the sweep leg.
|
|
#
|
|
# selftest 20 cells on fixture trees: discovery (a near-miss named CHECKER is
|
|
# not an entry point), the five codes the runner can produce
|
|
# (0, 1, 2, 124 timeout, python shebang), the four mutants the
|
|
# grader must name (new path, missing path, 0->1, 1->0), a baseline
|
|
# that is absent or malformed reading rc 2, and two takes being
|
|
# byte-identical.
|
|
# paths the entry-point SET, graded against the baseline, without running
|
|
# anything -- 0.2 s, and it is the leg that notices evidence leaving
|
|
# the tree.
|
|
# sweep every entry point run, its exit code compared with the code the
|
|
# baseline records, at the SAME --timeout 120 --jobs 4 the baseline
|
|
# is taken at (a 45 s take compared against a 120 s run calls every
|
|
# slow artifact MOVED). Both directions: a green that went red is a
|
|
# reverted repair, a red that went green is a repair nobody signed.
|
|
#
|
|
# WHERE THE TOOL LIVES
|
|
# This wrapper sits at the repo root (the house convention for a top-level
|
|
# gate), and its tool, baseline and report live three levels down in
|
|
# wizard-work/volt/q143-check-sweep/. The first version derived TOOL and BASE
|
|
# from $HERE -- i.e. from the repo root, where neither file exists -- so all
|
|
# three legs printed UNGRAD rc=2 and the entry point graded nothing at all
|
|
# (Q144). Both paths are named explicitly and their absence is rc 2.
|
|
#
|
|
# DEFAULT LEGS vs --full
|
|
# The default run is the two legs that are deterministic and fast (selftest
|
|
# 20 cells, paths 0.2 s): a repo-root gate that re-runs 97 entry points cannot
|
|
# be graded by the sweep that calls it, and a reading that depends on host load
|
|
# (REPORT §5) is not a gate. The full three-leg grade, including the sweep leg,
|
|
# is `bash CHECK-q143 --full` -- explicit, on an idle host.
|
|
#
|
|
# Exit codes: 0 all legs green, 1 a leg faulted, 2 a leg could not grade.
|
|
set -u
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$HERE"
|
|
ART="$HERE/wizard-work/volt/q143-check-sweep"
|
|
TOOL="$ART/check_sweep.py"
|
|
BASE="$ART/check_entry_baseline.tsv"
|
|
PY="${PYTHON:-python3}"
|
|
FULL=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--full) FULL=1 ;;
|
|
*) printf 'usage: %s [--full]\n' "$0" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
for f in "$TOOL" "$BASE"; do
|
|
if [ ! -f "$f" ]; then
|
|
printf 'CANNOT GRADE: missing %s\n' "$f"
|
|
exit 2
|
|
fi
|
|
done
|
|
rc=0
|
|
legs=0
|
|
faults=0
|
|
|
|
run_leg() {
|
|
name="$1"; shift
|
|
legs=$((legs + 1))
|
|
out="$("$PY" "$TOOL" --repo "$ROOT" --baseline "$BASE" "$@" 2>&1)"
|
|
lrc=$?
|
|
if [ "$lrc" -eq 0 ]; then
|
|
printf ' %-8s PASS rc=0 %s\n' "$name" "$(printf '%s' "$out" | tail -1)"
|
|
elif [ "$lrc" -eq 1 ]; then
|
|
faults=$((faults + 1))
|
|
printf ' %-8s FAULT rc=1 %s\n' "$name" "$(printf '%s' "$out" | tail -1)"
|
|
printf '%s\n' "$out" | grep -E '^(FAIL|!)' | head -8
|
|
else
|
|
faults=$((faults + 1))
|
|
printf ' %-8s UNGRAD rc=%s %s\n' "$name" "$lrc" "$(printf '%s' "$out" | tail -1)"
|
|
fi
|
|
[ "$lrc" -eq 0 ] || rc=1
|
|
}
|
|
|
|
run_leg selftest --selftest
|
|
run_leg paths --paths
|
|
if [ "$FULL" -eq 1 ]; then
|
|
# Same timeout and concurrency the baseline was taken at: a code recorded
|
|
# under a 45 s cap and compared against a 120 s run reports MOVED for every
|
|
# row that merely needed longer, which is noise rather than a move.
|
|
run_leg sweep --check --timeout 120 --jobs 4
|
|
fi
|
|
|
|
if [ "$rc" -eq 0 ]; then
|
|
echo "q143: $legs leg(s), all passed"
|
|
else
|
|
echo "q143: $legs leg(s), $faults fault(s)"
|
|
fi
|
|
exit "$rc"
|