130 lines
5.5 KiB
Python
130 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""q93_axis_negative_probe.py -- Q93's two negative cells, at the AXIS level.
|
|
|
|
Q93 (2026-10-27) changed the malformed-input axis of `tools/selftest_sweep.py`
|
|
from PROSE grading to CONTRACT grading, and gave the run one verdict line whose
|
|
text and exit code come from the same function. A green axis after such a change
|
|
proves only that the corpus agrees with the new rule; it does not prove the axis
|
|
can still FAIL. This probe supplies the two negative shapes Q93 asks for, through
|
|
the real subprocess path (`discover` -> `read_axis` -> `report_read`), against
|
|
scratch copies of `tools/`:
|
|
|
|
A answers-zero : a tool that exits 0 on a malformed input
|
|
EXPECT rc 1, malformed-input 1 fault, verdict RC_WRONG,
|
|
and `citation_hash_class.py/not-a-repo` named on the summary
|
|
line (Q93: UNEXPLAINED never appears unnamed).
|
|
B odd-prose : a tool that refuses correctly at rc 2 with prose no table
|
|
ever pinned
|
|
EXPECT rc 0, 0 fault -- the axis grades the contract, so a
|
|
sound refusal with different wording is no longer red. Under
|
|
the pre-Q93 table this same tool was UNEXPLAINED, which is
|
|
the defect: the axis would have gone green only if the tool's
|
|
message were degraded to match a pinned phrase.
|
|
|
|
Root is resolved from this file (`wizard-work/volt/` -> repo root), not written
|
|
down, so a second checkout re-verifies itself. Scratch copies live under $TMPDIR
|
|
and are removed. No file in the working tree is modified.
|
|
|
|
rc 0 both shapes behaved as stated (the axis can fail, and for the right reason)
|
|
rc 1 a shape behaved differently: the axis is blind, or still grades wording
|
|
rc 2 could not grade: no selftest_sweep.py, no tools/, or a scratch build failed
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
SWEEP = os.path.join(ROOT, "tools", "selftest_sweep.py")
|
|
TOOLS = os.path.join(ROOT, "tools")
|
|
|
|
SHIM_ZERO = ("import sys as _s, os as _o\n"
|
|
"if '--repo' in _s.argv:\n"
|
|
" _r = _s.argv[_s.argv.index('--repo') + 1]\n"
|
|
" if not _o.path.isdir(_o.path.join(_r, '.git')):\n"
|
|
" print('0 citations classified; nothing to do')\n"
|
|
" raise SystemExit(0)\n")
|
|
SHIM_REFUSE = ("import sys as _s, os as _o\n"
|
|
"if '--repo' in _s.argv:\n"
|
|
" _r = _s.argv[_s.argv.index('--repo') + 1]\n"
|
|
" if not _o.path.isdir(_o.path.join(_r, '.git')):\n"
|
|
" _s.stderr.write('totally different words, same refusal\\n')\n"
|
|
" raise SystemExit(2)\n")
|
|
|
|
|
|
def load_sweep():
|
|
spec = importlib.util.spec_from_file_location("q93_sweep", SWEEP)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod) # side-effect-free: guarded by __main__
|
|
return mod
|
|
|
|
|
|
def build(base: str, name: str, shim: str) -> str:
|
|
"""A copy of tools/ whose citation_hash_class.py opens with `shim`."""
|
|
d = os.path.join(base, name)
|
|
os.makedirs(d)
|
|
for fn in sorted(os.listdir(TOOLS)):
|
|
if fn.endswith(".py"):
|
|
shutil.copy(os.path.join(TOOLS, fn), os.path.join(d, fn))
|
|
target = os.path.join(d, "citation_hash_class.py")
|
|
with open(target, encoding="utf-8") as fh:
|
|
body = fh.read()
|
|
with open(target, "w", encoding="utf-8") as fh:
|
|
fh.write(shim + body)
|
|
return d
|
|
|
|
|
|
def probe(mod, tools_dir: str):
|
|
"""Run both input axes against `tools_dir`; return (rc, read, malformed,
|
|
malformed summary line, named faults)."""
|
|
rows, _source, _untracked = mod.discover(None, tools_dir)
|
|
axis = mod.read_axis(rows, 120)
|
|
named = [(r["tool"], r["probe"], r["verdict"]) for r in mod.axis_faults(axis)]
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
rc, read_fault, malformed_fault = mod.report_read(axis, quiet=True)
|
|
lines = [ln for ln in buf.getvalue().splitlines()
|
|
if ln.startswith("malformed-input:")]
|
|
return rc, read_fault, malformed_fault, (lines[0] if lines else ""), named
|
|
|
|
|
|
def main() -> int:
|
|
if not os.path.isfile(SWEEP):
|
|
print("cannot grade: no tools/selftest_sweep.py under " + ROOT)
|
|
return 2
|
|
if not os.path.isdir(TOOLS):
|
|
print("cannot grade: no tools/ under " + ROOT)
|
|
return 2
|
|
mod = load_sweep()
|
|
ok = True
|
|
with tempfile.TemporaryDirectory(prefix="q93neg.") as base:
|
|
d = build(base, "answers-zero", SHIM_ZERO)
|
|
rc, _rf, mf, line, named = probe(mod, d)
|
|
a_ok = (rc == 1 and mf == 1
|
|
and ("citation_hash_class.py", "not-a-repo", "RC_WRONG") in named
|
|
and "citation_hash_class.py/not-a-repo" in line)
|
|
ok &= a_ok
|
|
print("A answers-zero rc=%d malformed_fault=%d %s"
|
|
% (rc, mf, "EXPECTED" if a_ok else "UNEXPECTED"))
|
|
print(" %s" % line)
|
|
|
|
d = build(base, "odd-prose", SHIM_REFUSE)
|
|
rc, _rf, mf, line, named = probe(mod, d)
|
|
b_ok = (rc == 0 and mf == 0 and not named)
|
|
ok &= b_ok
|
|
print("B odd-prose rc=%d malformed_fault=%d %s"
|
|
% (rc, mf, "EXPECTED" if b_ok else "UNEXPECTED"))
|
|
print(" %s" % line)
|
|
|
|
print("q93 axis negative cells: %s" % ("2/2 as expected" if ok else "FAILED"))
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|