ledger
f864998aba
Q105 pointed beam/CHECK at pytest; pytest is not installed on this interpreter, so the gate exited 2 and ran zero cells. run_lane_tests.py collects TestCase classes (TestLoader) and bare module-level def test_*() (FunctionTestCase) itself, treats an unimportable module and a printed FAIL transcript as faults, and exits 0/1/2. Four test files that imported pytest now use beam_assert.raises/fail and unittest.mock; no cell dropped. Measured: unittest discover 353 tests; runner 570 (353 + 217 bare); bash wizard-work/beam/CHECK rc 0; test_run_lane_tests.py 10 cells OK; 7-mutant battery in scratch copies all as expected (HEAD CHECK rc 2, broken bare rc 1, pytest import rc 1, failing TestCase rc 1, script FAIL rc 1, empty lane rc 2, control rc 0). Record: WORK-QUEUE.md Q128; report wizard-outputs/volt/2026-10-30-q128.md
166 lines
7.0 KiB
Python
166 lines
7.0 KiB
Python
"""Q128: cells for run_lane_tests.py, the beam lane's own stdlib runner.
|
|
|
|
Each cell builds a throwaway lane in a temporary directory and runs the runner
|
|
as a subprocess with `--lane`, so the runner is graded by its exit code and its
|
|
printed summary, never by importing its internals. The cells that matter are
|
|
the red ones: a failing bare-style `def test_*`, a failing unittest cell, and a
|
|
module that will not import must each be a fault (rc 1), and an empty lane must
|
|
be rc 2 rather than a silent pass.
|
|
|
|
This file is itself collected by the runner it grades -- it is a plain
|
|
unittest.TestCase module, so it costs one extra cell set and no recursion
|
|
(the cells invoke the runner on temp lanes, never on this lane).
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
RUNNER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "run_lane_tests.py")
|
|
|
|
|
|
def run_lane(lane_dir):
|
|
"""Run the runner over ``lane_dir``; return (rc, stdout+stderr)."""
|
|
proc = subprocess.run(
|
|
[sys.executable, RUNNER, "--lane", lane_dir],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
return proc.returncode, proc.stdout
|
|
|
|
|
|
def write(path, text):
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write(text)
|
|
|
|
|
|
class TestRunnerContract(unittest.TestCase):
|
|
def setUp(self):
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.lane = self._tmp.name
|
|
|
|
def tearDown(self):
|
|
self._tmp.cleanup()
|
|
|
|
def test_green_lane_exit_zero_and_counts_both_styles(self):
|
|
write(os.path.join(self.lane, "test_a.py"),
|
|
"import unittest\n"
|
|
"def test_bare_ok():\n assert 1 + 1 == 2\n"
|
|
"class T(unittest.TestCase):\n"
|
|
" def test_uni_ok(self):\n self.assertEqual(2 + 2, 4)\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 0, out)
|
|
self.assertIn("1 unittest case(s), 1 bare case(s), 2 test(s)", out)
|
|
self.assertIn("lane: OK", out)
|
|
|
|
def test_failing_bare_cell_is_a_fault_named_by_module_and_function(self):
|
|
write(os.path.join(self.lane, "test_bare.py"),
|
|
"def test_broken():\n assert 1 == 2, 'bare cell broke'\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 1, out)
|
|
self.assertIn("FAIL: test_bare.test_broken", out)
|
|
self.assertIn("1 failure(s)", out)
|
|
|
|
def test_failing_unittest_cell_is_a_fault(self):
|
|
write(os.path.join(self.lane, "test_uni.py"),
|
|
"import unittest\n"
|
|
"class T(unittest.TestCase):\n"
|
|
" def test_broken(self):\n self.assertEqual(1, 2)\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 1, out)
|
|
self.assertIn("test_broken", out)
|
|
|
|
def test_module_that_will_not_import_is_a_fault_not_a_skip(self):
|
|
write(os.path.join(self.lane, "test_import_bang.py"),
|
|
"def broken(:\n pass\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 1, out)
|
|
self.assertIn("IMPORT-FAIL test_import_bang.py", out)
|
|
self.assertIn("1 import fault(s)", out)
|
|
|
|
def test_module_level_assertion_is_a_fault_too(self):
|
|
# A module that raises at import for a non-syntax reason (a failed
|
|
# module-level assertion in the lane's script-style files) is a fault
|
|
# too: the runner must not treat "no test_* names defined" as a pass.
|
|
write(os.path.join(self.lane, "test_script_style.py"),
|
|
"assert 1 == 2, 'module-level assertion ran'\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 1, out)
|
|
self.assertIn("IMPORT-FAIL test_script_style.py", out)
|
|
|
|
def test_script_that_prints_FAIL_is_a_fault(self):
|
|
# Q128: this lane's script-style files report a failed check by
|
|
# printing `FAIL <name>` and exiting 1 from their own __main__ guard.
|
|
# Under an import the guard never runs, so before this rule the
|
|
# runner called such a file green while its transcript said FAIL
|
|
# (measured on test_bisect_bounds.py: rc 0 with a false expectation).
|
|
write(os.path.join(self.lane, "test_script_says_fail.py"),
|
|
"ok = True\n"
|
|
"def check(name, cond):\n"
|
|
" global ok\n"
|
|
" print(('PASS' if cond else 'FAIL') + ' ' + name)\n"
|
|
" ok = ok and cond\n"
|
|
"check('a', True)\n"
|
|
"check('b', 1 == 2)\n"
|
|
"print('ALL' if ok else 'SOME FAILED')\n"
|
|
"if __name__ == '__main__':\n"
|
|
" raise SystemExit(0 if ok else 1)\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 1, out)
|
|
self.assertIn("SCRIPT-FAIL test_script_says_fail.py", out)
|
|
self.assertIn("FAIL b", out)
|
|
self.assertIn("1 script fault(s)", out)
|
|
|
|
def test_script_that_prints_only_PASS_is_not_a_fault(self):
|
|
# The negative control for the rule above: the same shape with every
|
|
# check true must stay green, so the cell proves the rule reads the
|
|
# transcript rather than firing on the convention itself.
|
|
write(os.path.join(self.lane, "test_script_says_pass.py"),
|
|
"ok = True\n"
|
|
"def check(name, cond):\n"
|
|
" global ok\n"
|
|
" print(('PASS' if cond else 'FAIL') + ' ' + name)\n"
|
|
" ok = ok and cond\n"
|
|
"check('a', True)\n"
|
|
"print('ALL' if ok else 'SOME FAILED')\n"
|
|
"if __name__ == '__main__':\n"
|
|
" raise SystemExit(0 if ok else 1)\n")
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 0, out)
|
|
self.assertNotIn("SCRIPT-FAIL", out)
|
|
self.assertIn("1 module-level script file(s)", out)
|
|
|
|
def test_no_test_files_is_could_not_grade(self):
|
|
rc, out = run_lane(self.lane)
|
|
self.assertEqual(rc, 2, out)
|
|
self.assertIn("nothing to grade", out)
|
|
|
|
def test_missing_lane_dir_is_could_not_grade(self):
|
|
rc, out = run_lane(os.path.join(self.lane, "does-not-exist"))
|
|
self.assertEqual(rc, 2, out)
|
|
self.assertIn("no lane directory", out)
|
|
|
|
def test_list_mode_prints_the_collection_and_runs_nothing(self):
|
|
write(os.path.join(self.lane, "test_c.py"),
|
|
"def test_one():\n assert True\n")
|
|
bad = os.path.join(self.lane, "test_d.py")
|
|
write(bad, "raise SystemExit('should not run in --list mode')\n")
|
|
proc = subprocess.run(
|
|
[sys.executable, RUNNER, "--lane", self.lane, "--list"],
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120,
|
|
)
|
|
# test_d.py raises SystemExit at import: --list must still report it as
|
|
# an import fault (rc 1) rather than dying or printing a clean count.
|
|
self.assertEqual(proc.returncode, 1, proc.stdout)
|
|
self.assertIn("IMPORT-FAIL", proc.stdout)
|
|
self.assertIn("test_d.py", proc.stdout)
|
|
self.assertIn("test_c.py", proc.stdout)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|