140 lines
5.4 KiB
Bash
140 lines
5.4 KiB
Bash
#!/usr/bin/env bash
|
|
# CHECK-q85 -- a lane module no collected cell reads is now a graded fault in
|
|
# beam's gate, and the six modules the census found invisible have cells
|
|
# (volt, 2026-11-02).
|
|
#
|
|
# Five legs:
|
|
# selftest check_coverage.py's own 13 cells: import / chain / path-literal
|
|
# coverage, an uncovered module named, a waiver with a reason, a
|
|
# waiver with none, a waiver naming a gone file, a waiver left
|
|
# over a covered module, a gate-invoked module, no-tests and
|
|
# unparseable-test rc 2, and the mutation cell (delete the one
|
|
# test -> 0 becomes 1)
|
|
# live the lane reads 0 today, and one planted module makes it 1 with
|
|
# the name printed, then 0 again when the plant is removed
|
|
# before the same tool over the lane at HEAD (no repair): rc 1, and the
|
|
# names it prints are the pre-repair census
|
|
# mutant Q85's unpinned docstring claim: M-JUL (the Julian leap branch
|
|
# dropped) reds test_zeller_dow; the unmutated copy is green
|
|
# gate the lane's own CHECK reads 0 with the coverage leg inside it
|
|
#
|
|
# Exit codes: 0 all legs passed, 1 a fault, 2 could not grade.
|
|
set -u
|
|
set -o pipefail
|
|
|
|
cd "$(dirname "${BASH_SOURCE[0]}")" || exit 2
|
|
ROOT="$(pwd)"
|
|
LANE="$ROOT/wizard-work/beam"
|
|
COV="$LANE/check_coverage.py"
|
|
WORK="$(mktemp -d "${TMPDIR:-/tmp}/check-q85.XXXXXX")" || exit 2
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
passed=0
|
|
failed=0
|
|
ungradable=0
|
|
row() { printf ' %-11s %s\n' "$1" "$2"; }
|
|
|
|
if [ ! -f "$COV" ]; then
|
|
echo "CHECK-q85: no $COV -- cannot grade" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# --- leg 1: the tool's selftest ---------------------------------------------
|
|
python3 "$COV" --selftest >"$WORK/st.out" 2>&1
|
|
st=$?
|
|
if [ "$st" -eq 0 ]; then
|
|
row selftest "PASS $(tail -n 1 "$WORK/st.out")"
|
|
passed=$((passed + 1))
|
|
else
|
|
row selftest "FAULT rc=$st (want 0)"
|
|
cat "$WORK/st.out"
|
|
failed=$((failed + 1))
|
|
fi
|
|
|
|
# --- leg 2: the live lane, and one planted module ---------------------------
|
|
mkdir -p "$WORK/live"
|
|
cp -a "$LANE/." "$WORK/live/"
|
|
( cd "$WORK/live" && python3 check_coverage.py >"$WORK/live1.out" 2>&1 )
|
|
l1=$?
|
|
printf '"""a module no collected test reads."""\n\n\ndef stray():\n return 1\n' \
|
|
>"$WORK/live/stray_module.py"
|
|
( cd "$WORK/live" && python3 check_coverage.py >"$WORK/live2.out" 2>&1 )
|
|
l2=$?
|
|
rm -f "$WORK/live/stray_module.py"
|
|
( cd "$WORK/live" && python3 check_coverage.py >"$WORK/live3.out" 2>&1 )
|
|
l3=$?
|
|
if [ "$l1" -eq 0 ] && [ "$l2" -eq 1 ] && [ "$l3" -eq 0 ] \
|
|
&& grep -q "UNCOVERED \+stray_module" "$WORK/live2.out"; then
|
|
row live "PASS lane 0, planted module 1 (named), removed 0"
|
|
passed=$((passed + 1))
|
|
else
|
|
row live "FAULT lane rc=$l1 (want 0), planted rc=$l2 (want 1), removed rc=$l3 (want 0)"
|
|
grep -E "UNCOVERED|^coverage:" "$WORK/live1.out" "$WORK/live2.out" "$WORK/live3.out"
|
|
failed=$((failed + 1))
|
|
fi
|
|
|
|
# --- leg 3: the census the repair answers -----------------------------------
|
|
# The no-repair baseline is e3f4875, the commit this repair closes against;
|
|
# grading it with this tool must go red and name the uncovered modules.
|
|
mkdir -p "$WORK/before"
|
|
git -C "$ROOT" archive e3f4875 wizard-work/beam | tar -x -C "$WORK/before" 2>"$WORK/tar.err"
|
|
tar_rc=$?
|
|
cp "$COV" "$WORK/before/wizard-work/beam/check_coverage.py"
|
|
( cd "$WORK/before/wizard-work/beam" && python3 check_coverage.py >"$WORK/before.out" 2>&1 )
|
|
b=$?
|
|
names="$(grep -c 'UNCOVERED ' "$WORK/before.out")"
|
|
if [ "$tar_rc" -eq 0 ] && [ "$b" -eq 1 ] && [ "$names" -ge 6 ]; then
|
|
row before "PASS pre-repair lane rc=1, $names module(s) named: $(grep 'UNCOVERED ' "$WORK/before.out" | awk '{print $2}' | tr '\n' ' ')"
|
|
passed=$((passed + 1))
|
|
elif [ "$tar_rc" -ne 0 ]; then
|
|
row before "UNGRADABLE git archive rc=$tar_rc"
|
|
cat "$WORK/tar.err"
|
|
ungradable=$((ungradable + 1))
|
|
else
|
|
row before "FAULT HEAD lane rc=$b (want 1), $names module(s) named (want >= 6)"
|
|
tail -3 "$WORK/before.out"
|
|
failed=$((failed + 1))
|
|
fi
|
|
|
|
# --- leg 4: the M-JUL mutant dies ------------------------------------------
|
|
mkdir -p "$WORK/mut"
|
|
cp -a "$LANE/." "$WORK/mut/"
|
|
( cd "$WORK/mut" && python3 -m unittest test_zeller_dow >"$WORK/m1.out" 2>&1 )
|
|
m1=$?
|
|
sed -i 's/^ leap = _julian_leap(y) if julian else _gregorian_leap(y)$/ leap = _gregorian_leap(y)/' \
|
|
"$WORK/mut/zeller_dow.py"
|
|
mutated="$(grep -c '^ leap = _gregorian_leap(y)$' "$WORK/mut/zeller_dow.py")"
|
|
( cd "$WORK/mut" && python3 -m unittest test_zeller_dow >"$WORK/m2.out" 2>&1 )
|
|
m2=$?
|
|
if [ "$m1" -eq 0 ] && [ "$mutated" -eq 1 ] && [ "$m2" -eq 1 ] \
|
|
&& grep -q "test_julian_leap_branch_is_the_witness" "$WORK/m2.out"; then
|
|
row mutant "PASS clean 0, M-JUL 1 (julian-branch cell names itself)"
|
|
passed=$((passed + 1))
|
|
else
|
|
row mutant "FAULT clean rc=$m1 (want 0), mutated rc=$m2 (want 1), patched lines=$mutated"
|
|
tail -5 "$WORK/m2.out"
|
|
failed=$((failed + 1))
|
|
fi
|
|
|
|
# --- leg 5: the lane gate, with the leg inside it ---------------------------
|
|
bash "$LANE/CHECK" >"$WORK/gate.out" 2>&1
|
|
g=$?
|
|
if [ "$g" -eq 0 ]; then
|
|
row gate "PASS $(grep -E '^lane: [0-9]+ file' "$WORK/gate.out" | head -1)"
|
|
passed=$((passed + 1))
|
|
elif [ "$g" -eq 2 ]; then
|
|
row gate "UNGRADABLE rc=2"
|
|
tail -5 "$WORK/gate.out"
|
|
ungradable=$((ungradable + 1))
|
|
else
|
|
row gate "FAULT rc=$g"
|
|
tail -15 "$WORK/gate.out"
|
|
failed=$((failed + 1))
|
|
fi
|
|
|
|
echo
|
|
echo "q85: $passed leg(s) passed, $failed fault(s), $ungradable ungradable"
|
|
[ "$failed" -eq 0 ] || exit 1
|
|
[ "$ungradable" -eq 0 ] || exit 2
|
|
exit 0
|