troupe-standup/wizard-work/beam/run_lane_tests.py

#!/usr/bin/env python3
"""run_lane_tests.py -- grade every test_*.py in this lane, stdlib only.

Why this exists (Q128, 2026-10-30).  The lane writes tests in two styles:
`unittest.TestCase` subclasses and bare module-level `def test_*()` functions
whose bodies are plain `assert`s.  `python3 -m unittest discover` collects only
the first style.  Q105's repair pointed `CHECK` at pytest, which collects both
-- and then the gate could not run at all on an interpreter without pytest:
`beam/CHECK` printed "pytest not importable -- cannot grade" and exited 2, so
the lane's 500-odd cells graded nothing while the gate was technically honest.

This runner collects both styles itself, in one process, with no third-party
import:

  * every top-level `test_*.py` is imported by file path under a unique module
    name, so two files sharing a basename cannot collide;
  * `unittest.TestLoader().loadTestsFromModule` picks up the TestCase classes;
  * each module-level function named `test_*` that takes no arguments is
    wrapped in `unittest.FunctionTestCase` and run in the same suite, so a
    failing bare assertion is a failing test, not silence;
  * a module that will not import is reported by name as a FAULT (rc 1), never
    skipped and never counted as a pass;
  * a module whose import-time transcript prints a failed check (`FAIL <name>`,
    or the `SOME FAILED` summary -- the `check()` convention this lane's
    script-style files use) is a SCRIPT-FAIL, rc 1: those files report by
    printing and only exit 1 from their own `__main__` guard, which never runs
    under an import, so the transcript is the only evidence they leave;
  * module-level stdout/stderr is captured during import and printed only when
    that import fails, so the summary is readable.

Exit codes follow the repo's tools/ contract:
  0  every collected test passed and every module imported
  1  a test failed, a test errored, or a module would not import (a fault)
  2  could not grade: no lane directory, no test files, or a bad argument

`--list` prints the collection (file, unittest cases, bare cases) and exits 0
without running anything, so the denominator can be quoted without a run.
"""

import argparse
import contextlib
import glob
import importlib.util
import inspect
import io
import os
import sys
import traceback
import unittest

DEFAULT_LANE = os.path.dirname(os.path.abspath(__file__))


def discover(lane):
    return sorted(glob.glob(os.path.join(lane, "test_*.py")))


def module_name(path, stem):
    """A sys.modules key for one test file: the file stem when free."""
    existing = sys.modules.get(stem)
    if existing is None:
        return stem
    if os.path.abspath(getattr(existing, "__file__", "") or "") == os.path.abspath(path):
        return stem
    return "_beam_lane_%s_%08x" % (stem, abs(hash(os.path.abspath(path))) % (1 << 32))


def load_module(path):
    """Import one test file by path; failure is raised as _ImportFault."""
    stem = os.path.splitext(os.path.basename(path))[0]
    modname = module_name(path, stem)
    spec = importlib.util.spec_from_file_location(modname, path)
    if spec is None or spec.loader is None:
        raise ImportError("no import spec for %s" % path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[modname] = module
    capture = io.StringIO()
    try:
        with contextlib.redirect_stdout(capture), contextlib.redirect_stderr(capture):
            spec.loader.exec_module(module)
    except BaseException as exc:  # noqa: BLE001 - any import-time failure is a fault
        tb_text = traceback.format_exc()
        sys.modules.pop(modname, None)
        raise _ImportFault(exc, tb_text, capture.getvalue())
    output = capture.getvalue()
    bad_line = transcript_failure(output)
    if bad_line is not None:
        # Q128: this lane's script-style files report a failed check by
        # printing `FAIL <name>` and keeping an `ok` flag, then exiting 1
        # from their own __main__ guard -- which never runs under an import.
        # Without this a broken script cell printed its FAIL line into a
        # buffer and the runner called the lane green (measured on
        # test_bisect_bounds.py: rc 0 with a false expectation in it).
        exc = AssertionError("module-level transcript reported a failure: %s" % bad_line)
        raise _ImportFault(exc, "", output, kind="SCRIPT-FAIL")
    return module, output


def transcript_failure(output):
    """The first line of a module-level transcript that reports a failure.

    Two forms, both taken from the lane's own script files rather than
    invented here: a line beginning `FAIL` (the `check()` helper every
    script file defines prints `PASS <name>` / `FAIL <name>`), and the
    `SOME FAILED` summary.  `PASS`, `ALL`, `OK` and prose are not failures,
    and a line is only read as one at its first word, so `preFAIL` or a
    path containing FAIL is not a false positive.
    """
    for line in output.splitlines():
        token = line.strip()
        if not token:
            continue
        first = token.split(None, 1)[0].rstrip(":")
        if first == "FAIL" or token.endswith("SOME FAILED"):
            return token
    return None


class _ImportFault(Exception):
    def __init__(self, exc, tb_text, output, kind="IMPORT-FAIL"):
        Exception.__init__(self, "%s: %s" % (type(exc).__name__, exc))
        self.exc = exc
        self.tb_text = tb_text
        self.output = output
        self.kind = kind


class BareCase(unittest.FunctionTestCase):
    """A module-level ``def test_*()`` presented like any other test."""

    def __init__(self, fn, label):
        unittest.FunctionTestCase.__init__(self, fn)
        self._label = label
        self._name = fn.__name__

    def id(self):
        return "%s.%s" % (self._label, self._name)

    def __str__(self):
        return self.id()


def bare_cases(module, modname, label):
    """Module-level zero-argument functions named test_* -> runnable cases."""
    cases = []
    for name, obj in sorted(vars(module).items()):
        if not name.startswith("test_"):
            continue
        if not inspect.isfunction(obj):
            continue
        if getattr(obj, "__module__", None) != modname:
            continue  # imported, not defined here
        try:
            params = inspect.signature(obj).parameters
        except (TypeError, ValueError):
            continue
        if params:
            continue  # fixture-style signature: not this runner's business
        cases.append(BareCase(obj, label))
    return cases


def collect(lane):
    """Return (suite, report) where report is a list of per-file rows."""
    suite = unittest.TestSuite()
    rows = []
    faults = []
    for path in discover(lane):
        name = os.path.basename(path)
        try:
            module, _out = load_module(path)
        except _ImportFault as fault:
            faults.append((name, fault))
            rows.append({"file": name, "unittest": 0, "bare": 0, "fault": fault})
            continue
        modname = module.__name__
        uni = unittest.TestLoader().loadTestsFromModule(module)
        uni_count = uni.countTestCases()
        bare = bare_cases(module, modname, os.path.splitext(name)[0])
        suite.addTests(uni)
        suite.addTests(bare)
        rows.append({"file": name, "unittest": uni_count, "bare": len(bare), "fault": None})
    return suite, rows


def main(argv=None):
    ap = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0])
    ap.add_argument("--lane", default=DEFAULT_LANE, help="lane directory (default: this file's)")
    ap.add_argument("--list", action="store_true", help="print the collection and exit")
    ap.add_argument("--verbose", "-v", action="store_true", help="one line per test")
    args = ap.parse_args(argv)

    lane = os.path.abspath(args.lane)
    if not os.path.isdir(lane):
        print("run_lane_tests: no lane directory: %s" % lane, file=sys.stderr)
        return 2
    files = discover(lane)
    if not files:
        print("run_lane_tests: no test_*.py under %s -- nothing to grade" % lane, file=sys.stderr)
        return 2

    sys.path.insert(0, lane)
    cwd = os.getcwd()
    os.chdir(lane)
    try:
        suite, rows = collect(lane)
    finally:
        os.chdir(cwd)

    total_uni = sum(r["unittest"] for r in rows)
    total_bare = sum(r["bare"] for r in rows)
    scripts = [r["file"] for r in rows
               if r["fault"] is None and r["unittest"] == 0 and r["bare"] == 0]
    faults = [r for r in rows if r["fault"] is not None]
    import_faults = [r for r in faults if r["fault"].kind == "IMPORT-FAIL"]
    script_faults = [r for r in faults if r["fault"].kind == "SCRIPT-FAIL"]
    total = suite.countTestCases()

    if args.list:
        for r in rows:
            print("%-46s unittest=%-4d bare=%-4d%s"
                  % (r["file"], r["unittest"], r["bare"],
                     "  " + r["fault"].kind if r["fault"] else ""))
        print("collection: %d file(s), %d unittest case(s), %d bare case(s), %d total, "
              "%d module-level script file(s), %d import fault(s), %d script fault(s)"
              % (len(rows), total_uni, total_bare, total, len(scripts),
                 len(import_faults), len(script_faults)))
        return 1 if faults else 0

    runner = unittest.TextTestRunner(verbosity=2 if args.verbose else 1)
    result = runner.run(suite)

    bad = (not result.wasSuccessful()) or bool(faults)
    for r in faults:
        fault = r["fault"]
        print("\n%s %s" % (fault.kind, r["file"]), file=sys.stderr)
        print("  %s" % fault, file=sys.stderr)
        if fault.output.strip():
            tail = fault.output.strip().splitlines()[-3:]
            for line in tail:
                print("  import output: %s" % line, file=sys.stderr)
        if fault.tb_text:
            print("  traceback (last 6 lines):", file=sys.stderr)
            for line in fault.tb_text.strip().splitlines()[-6:]:
                print("  " + line, file=sys.stderr)

    print("lane: %d file(s), %d unittest case(s), %d bare case(s), %d test(s), "
          "%d module-level script file(s) executed at import, "
          "%d failure(s), %d error(s), %d import fault(s), %d script fault(s)"
          % (len(rows), total_uni, total_bare, total, len(scripts),
             len(result.failures), len(result.errors),
             len(import_faults), len(script_faults)))
    if bad:
        print("lane: FAULT -- see the failure list above (an unimportable module and a "
              "failed script transcript are faults, not skips)")
        return 1
    print("lane: OK -- every collected test passed")
    return 0


if __name__ == "__main__":
    sys.exit(main())