"""Q128: cells for run_lane_tests.py, the beam lane's own stdlib runner. Each cell builds a throwaway lane in a temporary directory and runs the runner as a subprocess with `--lane`, so the runner is graded by its exit code and its printed summary, never by importing its internals. The cells that matter are the red ones: a failing bare-style `def test_*`, a failing unittest cell, and a module that will not import must each be a fault (rc 1), and an empty lane must be rc 2 rather than a silent pass. This file is itself collected by the runner it grades -- it is a plain unittest.TestCase module, so it costs one extra cell set and no recursion (the cells invoke the runner on temp lanes, never on this lane). """ import os import subprocess import sys import tempfile import unittest RUNNER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "run_lane_tests.py") def run_lane(lane_dir): """Run the runner over ``lane_dir``; return (rc, stdout+stderr).""" proc = subprocess.run( [sys.executable, RUNNER, "--lane", lane_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120, ) return proc.returncode, proc.stdout def write(path, text): with open(path, "w", encoding="utf-8") as fh: fh.write(text) class TestRunnerContract(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory() self.lane = self._tmp.name def tearDown(self): self._tmp.cleanup() def test_green_lane_exit_zero_and_counts_both_styles(self): write(os.path.join(self.lane, "test_a.py"), "import unittest\n" "def test_bare_ok():\n assert 1 + 1 == 2\n" "class T(unittest.TestCase):\n" " def test_uni_ok(self):\n self.assertEqual(2 + 2, 4)\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 0, out) self.assertIn("1 unittest case(s), 1 bare case(s), 2 test(s)", out) self.assertIn("lane: OK", out) def test_failing_bare_cell_is_a_fault_named_by_module_and_function(self): write(os.path.join(self.lane, "test_bare.py"), "def test_broken():\n assert 1 == 2, 'bare cell broke'\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 1, out) self.assertIn("FAIL: test_bare.test_broken", out) self.assertIn("1 failure(s)", out) def test_failing_unittest_cell_is_a_fault(self): write(os.path.join(self.lane, "test_uni.py"), "import unittest\n" "class T(unittest.TestCase):\n" " def test_broken(self):\n self.assertEqual(1, 2)\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 1, out) self.assertIn("test_broken", out) def test_module_that_will_not_import_is_a_fault_not_a_skip(self): write(os.path.join(self.lane, "test_import_bang.py"), "def broken(:\n pass\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 1, out) self.assertIn("IMPORT-FAIL test_import_bang.py", out) self.assertIn("1 import fault(s)", out) def test_module_level_assertion_is_a_fault_too(self): # A module that raises at import for a non-syntax reason (a failed # module-level assertion in the lane's script-style files) is a fault # too: the runner must not treat "no test_* names defined" as a pass. write(os.path.join(self.lane, "test_script_style.py"), "assert 1 == 2, 'module-level assertion ran'\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 1, out) self.assertIn("IMPORT-FAIL test_script_style.py", out) def test_script_that_prints_FAIL_is_a_fault(self): # Q128: this lane's script-style files report a failed check by # printing `FAIL ` and exiting 1 from their own __main__ guard. # Under an import the guard never runs, so before this rule the # runner called such a file green while its transcript said FAIL # (measured on test_bisect_bounds.py: rc 0 with a false expectation). write(os.path.join(self.lane, "test_script_says_fail.py"), "ok = True\n" "def check(name, cond):\n" " global ok\n" " print(('PASS' if cond else 'FAIL') + ' ' + name)\n" " ok = ok and cond\n" "check('a', True)\n" "check('b', 1 == 2)\n" "print('ALL' if ok else 'SOME FAILED')\n" "if __name__ == '__main__':\n" " raise SystemExit(0 if ok else 1)\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 1, out) self.assertIn("SCRIPT-FAIL test_script_says_fail.py", out) self.assertIn("FAIL b", out) self.assertIn("1 script fault(s)", out) def test_script_that_prints_only_PASS_is_not_a_fault(self): # The negative control for the rule above: the same shape with every # check true must stay green, so the cell proves the rule reads the # transcript rather than firing on the convention itself. write(os.path.join(self.lane, "test_script_says_pass.py"), "ok = True\n" "def check(name, cond):\n" " global ok\n" " print(('PASS' if cond else 'FAIL') + ' ' + name)\n" " ok = ok and cond\n" "check('a', True)\n" "print('ALL' if ok else 'SOME FAILED')\n" "if __name__ == '__main__':\n" " raise SystemExit(0 if ok else 1)\n") rc, out = run_lane(self.lane) self.assertEqual(rc, 0, out) self.assertNotIn("SCRIPT-FAIL", out) self.assertIn("1 module-level script file(s)", out) def test_no_test_files_is_could_not_grade(self): rc, out = run_lane(self.lane) self.assertEqual(rc, 2, out) self.assertIn("nothing to grade", out) def test_missing_lane_dir_is_could_not_grade(self): rc, out = run_lane(os.path.join(self.lane, "does-not-exist")) self.assertEqual(rc, 2, out) self.assertIn("no lane directory", out) def test_list_mode_prints_the_collection_and_runs_nothing(self): write(os.path.join(self.lane, "test_c.py"), "def test_one():\n assert True\n") bad = os.path.join(self.lane, "test_d.py") write(bad, "raise SystemExit('should not run in --list mode')\n") proc = subprocess.run( [sys.executable, RUNNER, "--lane", self.lane, "--list"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120, ) # test_d.py raises SystemExit at import: --list must still report it as # an import fault (rc 1) rather than dying or printing a clean count. self.assertEqual(proc.returncode, 1, proc.stdout) self.assertIn("IMPORT-FAIL", proc.stdout) self.assertIn("test_d.py", proc.stdout) self.assertIn("test_c.py", proc.stdout) if __name__ == "__main__": unittest.main(verbosity=2)