#!/usr/bin/env python3 """run_lane_tests.py -- grade every test_*.py in this lane, stdlib only. Why this exists (Q128, 2026-10-30). The lane writes tests in two styles: `unittest.TestCase` subclasses and bare module-level `def test_*()` functions whose bodies are plain `assert`s. `python3 -m unittest discover` collects only the first style. Q105's repair pointed `CHECK` at pytest, which collects both -- and then the gate could not run at all on an interpreter without pytest: `beam/CHECK` printed "pytest not importable -- cannot grade" and exited 2, so the lane's 500-odd cells graded nothing while the gate was technically honest. This runner collects both styles itself, in one process, with no third-party import: * every top-level `test_*.py` is imported by file path under a unique module name, so two files sharing a basename cannot collide; * `unittest.TestLoader().loadTestsFromModule` picks up the TestCase classes; * each module-level function named `test_*` that takes no arguments is wrapped in `unittest.FunctionTestCase` and run in the same suite, so a failing bare assertion is a failing test, not silence; * a module that will not import is reported by name as a FAULT (rc 1), never skipped and never counted as a pass; * a module whose import-time transcript prints a failed check (`FAIL `, or the `SOME FAILED` summary -- the `check()` convention this lane's script-style files use) is a SCRIPT-FAIL, rc 1: those files report by printing and only exit 1 from their own `__main__` guard, which never runs under an import, so the transcript is the only evidence they leave; * module-level stdout/stderr is captured during import and printed only when that import fails, so the summary is readable. Exit codes follow the repo's tools/ contract: 0 every collected test passed and every module imported 1 a test failed, a test errored, or a module would not import (a fault) 2 could not grade: no lane directory, no test files, or a bad argument `--list` prints the collection (file, unittest cases, bare cases) and exits 0 without running anything, so the denominator can be quoted without a run. """ import argparse import contextlib import glob import importlib.util import inspect import io import os import sys import traceback import unittest DEFAULT_LANE = os.path.dirname(os.path.abspath(__file__)) def discover(lane): return sorted(glob.glob(os.path.join(lane, "test_*.py"))) def module_name(path, stem): """A sys.modules key for one test file: the file stem when free.""" existing = sys.modules.get(stem) if existing is None: return stem if os.path.abspath(getattr(existing, "__file__", "") or "") == os.path.abspath(path): return stem return "_beam_lane_%s_%08x" % (stem, abs(hash(os.path.abspath(path))) % (1 << 32)) def load_module(path): """Import one test file by path; failure is raised as _ImportFault.""" stem = os.path.splitext(os.path.basename(path))[0] modname = module_name(path, stem) spec = importlib.util.spec_from_file_location(modname, path) if spec is None or spec.loader is None: raise ImportError("no import spec for %s" % path) module = importlib.util.module_from_spec(spec) sys.modules[modname] = module capture = io.StringIO() try: with contextlib.redirect_stdout(capture), contextlib.redirect_stderr(capture): spec.loader.exec_module(module) except BaseException as exc: # noqa: BLE001 - any import-time failure is a fault tb_text = traceback.format_exc() sys.modules.pop(modname, None) raise _ImportFault(exc, tb_text, capture.getvalue()) output = capture.getvalue() bad_line = transcript_failure(output) if bad_line is not None: # Q128: this lane's script-style files report a failed check by # printing `FAIL ` and keeping an `ok` flag, then exiting 1 # from their own __main__ guard -- which never runs under an import. # Without this a broken script cell printed its FAIL line into a # buffer and the runner called the lane green (measured on # test_bisect_bounds.py: rc 0 with a false expectation in it). exc = AssertionError("module-level transcript reported a failure: %s" % bad_line) raise _ImportFault(exc, "", output, kind="SCRIPT-FAIL") return module, output def transcript_failure(output): """The first line of a module-level transcript that reports a failure. Two forms, both taken from the lane's own script files rather than invented here: a line beginning `FAIL` (the `check()` helper every script file defines prints `PASS ` / `FAIL `), and the `SOME FAILED` summary. `PASS`, `ALL`, `OK` and prose are not failures, and a line is only read as one at its first word, so `preFAIL` or a path containing FAIL is not a false positive. """ for line in output.splitlines(): token = line.strip() if not token: continue first = token.split(None, 1)[0].rstrip(":") if first == "FAIL" or token.endswith("SOME FAILED"): return token return None class _ImportFault(Exception): def __init__(self, exc, tb_text, output, kind="IMPORT-FAIL"): Exception.__init__(self, "%s: %s" % (type(exc).__name__, exc)) self.exc = exc self.tb_text = tb_text self.output = output self.kind = kind class BareCase(unittest.FunctionTestCase): """A module-level ``def test_*()`` presented like any other test.""" def __init__(self, fn, label): unittest.FunctionTestCase.__init__(self, fn) self._label = label self._name = fn.__name__ def id(self): return "%s.%s" % (self._label, self._name) def __str__(self): return self.id() def bare_cases(module, modname, label): """Module-level zero-argument functions named test_* -> runnable cases.""" cases = [] for name, obj in sorted(vars(module).items()): if not name.startswith("test_"): continue if not inspect.isfunction(obj): continue if getattr(obj, "__module__", None) != modname: continue # imported, not defined here try: params = inspect.signature(obj).parameters except (TypeError, ValueError): continue if params: continue # fixture-style signature: not this runner's business cases.append(BareCase(obj, label)) return cases def collect(lane): """Return (suite, report) where report is a list of per-file rows.""" suite = unittest.TestSuite() rows = [] faults = [] for path in discover(lane): name = os.path.basename(path) try: module, _out = load_module(path) except _ImportFault as fault: faults.append((name, fault)) rows.append({"file": name, "unittest": 0, "bare": 0, "fault": fault}) continue modname = module.__name__ uni = unittest.TestLoader().loadTestsFromModule(module) uni_count = uni.countTestCases() bare = bare_cases(module, modname, os.path.splitext(name)[0]) suite.addTests(uni) suite.addTests(bare) rows.append({"file": name, "unittest": uni_count, "bare": len(bare), "fault": None}) return suite, rows def main(argv=None): ap = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) ap.add_argument("--lane", default=DEFAULT_LANE, help="lane directory (default: this file's)") ap.add_argument("--list", action="store_true", help="print the collection and exit") ap.add_argument("--verbose", "-v", action="store_true", help="one line per test") args = ap.parse_args(argv) lane = os.path.abspath(args.lane) if not os.path.isdir(lane): print("run_lane_tests: no lane directory: %s" % lane, file=sys.stderr) return 2 files = discover(lane) if not files: print("run_lane_tests: no test_*.py under %s -- nothing to grade" % lane, file=sys.stderr) return 2 sys.path.insert(0, lane) cwd = os.getcwd() os.chdir(lane) try: suite, rows = collect(lane) finally: os.chdir(cwd) total_uni = sum(r["unittest"] for r in rows) total_bare = sum(r["bare"] for r in rows) scripts = [r["file"] for r in rows if r["fault"] is None and r["unittest"] == 0 and r["bare"] == 0] faults = [r for r in rows if r["fault"] is not None] import_faults = [r for r in faults if r["fault"].kind == "IMPORT-FAIL"] script_faults = [r for r in faults if r["fault"].kind == "SCRIPT-FAIL"] total = suite.countTestCases() if args.list: for r in rows: print("%-46s unittest=%-4d bare=%-4d%s" % (r["file"], r["unittest"], r["bare"], " " + r["fault"].kind if r["fault"] else "")) print("collection: %d file(s), %d unittest case(s), %d bare case(s), %d total, " "%d module-level script file(s), %d import fault(s), %d script fault(s)" % (len(rows), total_uni, total_bare, total, len(scripts), len(import_faults), len(script_faults))) return 1 if faults else 0 runner = unittest.TextTestRunner(verbosity=2 if args.verbose else 1) result = runner.run(suite) bad = (not result.wasSuccessful()) or bool(faults) for r in faults: fault = r["fault"] print("\n%s %s" % (fault.kind, r["file"]), file=sys.stderr) print(" %s" % fault, file=sys.stderr) if fault.output.strip(): tail = fault.output.strip().splitlines()[-3:] for line in tail: print(" import output: %s" % line, file=sys.stderr) if fault.tb_text: print(" traceback (last 6 lines):", file=sys.stderr) for line in fault.tb_text.strip().splitlines()[-6:]: print(" " + line, file=sys.stderr) print("lane: %d file(s), %d unittest case(s), %d bare case(s), %d test(s), " "%d module-level script file(s) executed at import, " "%d failure(s), %d error(s), %d import fault(s), %d script fault(s)" % (len(rows), total_uni, total_bare, total, len(scripts), len(result.failures), len(result.errors), len(import_faults), len(script_faults))) if bad: print("lane: FAULT -- see the failure list above (an unimportable module and a " "failed script transcript are faults, not skips)") return 1 print("lane: OK -- every collected test passed") return 0 if __name__ == "__main__": sys.exit(main())