#!/usr/bin/env python3 """Tests for checklint. Run directly: python3 test_checklint.py Exit code IS the verdict: 0 = every test passed, 1 = a test failed. The suite is the negative control for the linter itself: if checklint.py regresses to "always clean", the MISSING/FALLBACK_ARM/METACHAR/TAUTOLOGY cases below fail, and this file exits 1. Two legs are not static and are exercised for real: --run executes the extracted command (the sentinel is created by the command, not by the test), and --scan walks a directory tree built in a temp dir. """ from __future__ import annotations import json import subprocess import sys import tempfile import unittest from pathlib import Path HERE = Path(__file__).resolve().parent TOOL = HERE / "checklint.py" sys.path.insert(0, str(HERE)) import checklint # noqa: E402 CLEAN = "CHECK: bash wizard-work/null/CHECK\n" def entry(body: str) -> str: return "# entry\n\nsome prose\n\n" + body class StaticLintTests(unittest.TestCase): def verdict(self, text: str) -> tuple[list[checklint.Finding], int]: return checklint.grade(text) def codes(self, text: str) -> list[str]: return [f.code for f in self.verdict(text)[0]] # -- the contract holds ------------------------------------------------ def test_clean_bare_command_passes(self): findings, v = self.verdict(entry(CLEAN)) self.assertEqual(findings, []) self.assertEqual(v, 0) def test_absolute_path_command_passes(self): findings, v = self.verdict(entry("CHECK: python3 /root/x/test_gate.py\n")) self.assertEqual((findings, v), ([], 0)) def test_quoted_argument_without_metachars_passes(self): # a quoted path is not a control operator; the exit code is still the command's findings, v = self.verdict(entry('CHECK: python3 "/root/x/my gate.py"\n')) self.assertEqual((findings, v), ([], 0)) # -- MISSING / MULTIPLE / EMPTY --------------------------------------- def test_missing_check_line(self): findings, v = self.verdict("# entry\n\nno check here\n") self.assertEqual([f.code for f in findings], ["MISSING"]) self.assertEqual(v, 1) def test_two_check_lines(self): findings, v = self.verdict(entry("CHECK: python3 a.py\n\nCHECK: python3 b.py\n")) self.assertEqual([f.code for f in findings], ["MULTIPLE"]) self.assertEqual(v, 1) def test_empty_check_line(self): findings, v = self.verdict(entry("CHECK:\n")) self.assertEqual([f.code for f in findings], ["EMPTY"]) self.assertEqual(v, 1) def test_check_inside_fence_is_not_the_contract(self): text = entry("```\nCHECK: true\n```\n") self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"]) def test_prose_starting_with_check_colon_is_not_matched(self): text = entry("Note: CHECK: this is prose, not a contract line\n") self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"]) # -- fallback arms ----------------------------------------------------- def test_fallback_or_true(self): self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || true\n"))) def test_fallback_or_exit_0(self): self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || exit 0\n"))) def test_fallback_semicolon_exit_0(self): self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py; exit 0\n"))) def test_fallback_or_echo(self): self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || echo ok\n"))) def test_fallback_is_reported_as_forced_exit_not_as_a_metachar_only(self): findings = self.verdict(entry("CHECK: python3 a.py || true\n"))[0] self.assertTrue(any("cannot fail" in f.message for f in findings)) # -- tautologies ------------------------------------------------------ def test_tautology_true(self): self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: true\n"))) def test_tautology_exit_0(self): self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: exit 0\n"))) def test_tautology_echo(self): self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: echo ok\n"))) # -- metacharacters ---------------------------------------------------- def test_metachar_and_chain(self): self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py && python3 b.py\n"))) def test_metachar_pipe(self): self.assertIn("METACHAR", self.codes(entry("CHECK: cat f | grep x\n"))) def test_metachar_redirect(self): self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py >/dev/null 2>&1\n"))) def test_metachar_command_substitution(self): self.assertIn("METACHAR", self.codes(entry("CHECK: python3 $(ls)\n"))) def test_metachar_backtick(self): self.assertIn("METACHAR", self.codes(entry("CHECK: python3 `ls`\n"))) def test_metachar_line_continuation(self): self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py \\\n"))) # -- real corpus shapes, copied verbatim -------------------------------- def test_real_corpus_and_chain_shape_is_flagged(self): self.assertIn("METACHAR", self.codes(entry( "CHECK: test -s tools/ok_gate.py && grep -q VERDICT tools/ok_gate.py\n"))) def test_real_corpus_redirect_shape_is_flagged(self): self.assertIn("METACHAR", self.codes(entry( "CHECK: python3 /root/troupe-standup/wizard-work/forge/verify.py >/dev/null 2>&1\n"))) # -- purity ------------------------------------------------------------ def test_lint_text_is_pure_and_never_executes(self): with tempfile.TemporaryDirectory() as td: sentinel = Path(td) / "ran" text = entry(f"CHECK: touch {sentinel}\n") first = checklint.lint_text(text) second = checklint.lint_text(text) self.assertEqual(first, second) # pure: same input, same findings self.assertEqual(first, []) # statically clean self.assertFalse(sentinel.exists()) # and it did not execute anything class CliTests(unittest.TestCase): def run_tool(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: return subprocess.run( [sys.executable, str(TOOL), *args], capture_output=True, text=True, cwd=str(cwd) if cwd else None, timeout=120, ) def test_cli_clean_entry_exits_0(self): with tempfile.TemporaryDirectory() as td: p = Path(td) / "e.md" p.write_text(entry(CLEAN), encoding="utf-8") r = self.run_tool(str(p)) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) self.assertIn("OK", r.stdout) def test_cli_faulty_entry_exits_1(self): with tempfile.TemporaryDirectory() as td: p = Path(td) / "e.md" p.write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8") r = self.run_tool(str(p)) self.assertEqual(r.returncode, 1, r.stdout + r.stderr) self.assertIn("FALLBACK_ARM", r.stdout) def test_cli_unreadable_entry_exits_2(self): r = self.run_tool("/root/definitely/not/here.md") self.assertEqual(r.returncode, 2) self.assertIn("UNREADABLE", r.stderr) def test_cli_run_leg_executes_the_command(self): with tempfile.TemporaryDirectory() as td: sentinel = Path(td) / "ran" p = Path(td) / "e.md" p.write_text(entry(f"CHECK: touch {sentinel}\n"), encoding="utf-8") r = self.run_tool("--run", str(p)) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) self.assertTrue(sentinel.exists(), "the CHECK command was not executed") def test_cli_run_leg_reports_a_failing_command(self): with tempfile.TemporaryDirectory() as td: p = Path(td) / "e.md" p.write_text(entry("CHECK: false\n"), encoding="utf-8") r = self.run_tool("--run", str(p)) self.assertEqual(r.returncode, 1) self.assertIn("RUN_FAILURE", r.stdout) self.assertIn("exit 1", r.stdout) def test_cli_run_leg_does_not_execute_a_fallback_arm(self): with tempfile.TemporaryDirectory() as td: sentinel = Path(td) / "ran" p = Path(td) / "e.md" p.write_text(entry(f"CHECK: touch {sentinel} || true\n"), encoding="utf-8") r = self.run_tool("--run", str(p)) self.assertEqual(r.returncode, 1) self.assertFalse(sentinel.exists()) def test_cli_timeout_exits_2_and_names_its_cause(self): with tempfile.TemporaryDirectory() as td: p = Path(td) / "e.md" p.write_text(entry("CHECK: sleep 30\n"), encoding="utf-8") r = self.run_tool("--run", "--timeout", "0.5", str(p)) self.assertEqual(r.returncode, 2, r.stdout + r.stderr) self.assertIn("TIMEOUT", r.stdout) def test_cli_json_shape(self): with tempfile.TemporaryDirectory() as td: p = Path(td) / "e.md" p.write_text(entry("CHECK: true\n"), encoding="utf-8") r = self.run_tool("--json", str(p)) self.assertEqual(r.returncode, 1) data = json.loads(r.stdout) self.assertEqual(data["verdict"], 1) self.assertEqual(data["findings"][0]["code"], "TAUTOLOGY") self.assertIn("line", data["findings"][0]) def test_cli_no_entry_exits_2(self): r = self.run_tool() self.assertEqual(r.returncode, 2) class ScanTests(unittest.TestCase): def test_scan_gates_a_corpus(self): with tempfile.TemporaryDirectory() as td: root = Path(td) (root / "clean.md").write_text(entry(CLEAN), encoding="utf-8") (root / "faulty.md").write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8") (root / "no-check.md").write_text("# nothing\n", encoding="utf-8") (root / "bin").write_bytes(b"\x00\x01\x02") r = subprocess.run( [sys.executable, str(TOOL), "--scan", str(root)], capture_output=True, text=True, timeout=120, ) self.assertEqual(r.returncode, 1, r.stdout + r.stderr) self.assertIn("2 file(s) with a CHECK line, 1 clean, 1 faulty", r.stdout) self.assertIn("1 unreadable/binary skipped", r.stdout) def test_scan_clean_tree_exits_0(self): with tempfile.TemporaryDirectory() as td: root = Path(td) (root / "clean.md").write_text(entry(CLEAN), encoding="utf-8") r = subprocess.run( [sys.executable, str(TOOL), "--scan", str(root)], capture_output=True, text=True, timeout=120, ) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) def test_scan_unreadable_root_exits_2(self): r = subprocess.run( [sys.executable, str(TOOL), "--scan", "/root/definitely/not/a/dir"], capture_output=True, text=True, timeout=120, ) self.assertEqual(r.returncode, 2) if __name__ == "__main__": unittest.main(verbosity=2)