266 lines
11 KiB
Python
266 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for checklint. Run directly:
|
|
|
|
python3 test_checklint.py
|
|
|
|
Exit code IS the verdict: 0 = every test passed, 1 = a test failed.
|
|
The suite is the negative control for the linter itself: if checklint.py
|
|
regresses to "always clean", the MISSING/FALLBACK_ARM/METACHAR/TAUTOLOGY
|
|
cases below fail, and this file exits 1.
|
|
|
|
Two legs are not static and are exercised for real: --run executes the
|
|
extracted command (the sentinel is created by the command, not by the test),
|
|
and --scan walks a directory tree built in a temp dir.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
TOOL = HERE / "checklint.py"
|
|
sys.path.insert(0, str(HERE))
|
|
|
|
import checklint # noqa: E402
|
|
|
|
CLEAN = "CHECK: bash wizard-work/null/CHECK\n"
|
|
|
|
|
|
def entry(body: str) -> str:
|
|
return "# entry\n\nsome prose\n\n" + body
|
|
|
|
|
|
class StaticLintTests(unittest.TestCase):
|
|
def verdict(self, text: str) -> tuple[list[checklint.Finding], int]:
|
|
return checklint.grade(text)
|
|
|
|
def codes(self, text: str) -> list[str]:
|
|
return [f.code for f in self.verdict(text)[0]]
|
|
|
|
# -- the contract holds ------------------------------------------------
|
|
def test_clean_bare_command_passes(self):
|
|
findings, v = self.verdict(entry(CLEAN))
|
|
self.assertEqual(findings, [])
|
|
self.assertEqual(v, 0)
|
|
|
|
def test_absolute_path_command_passes(self):
|
|
findings, v = self.verdict(entry("CHECK: python3 /root/x/test_gate.py\n"))
|
|
self.assertEqual((findings, v), ([], 0))
|
|
|
|
def test_quoted_argument_without_metachars_passes(self):
|
|
# a quoted path is not a control operator; the exit code is still the command's
|
|
findings, v = self.verdict(entry('CHECK: python3 "/root/x/my gate.py"\n'))
|
|
self.assertEqual((findings, v), ([], 0))
|
|
|
|
# -- MISSING / MULTIPLE / EMPTY ---------------------------------------
|
|
def test_missing_check_line(self):
|
|
findings, v = self.verdict("# entry\n\nno check here\n")
|
|
self.assertEqual([f.code for f in findings], ["MISSING"])
|
|
self.assertEqual(v, 1)
|
|
|
|
def test_two_check_lines(self):
|
|
findings, v = self.verdict(entry("CHECK: python3 a.py\n\nCHECK: python3 b.py\n"))
|
|
self.assertEqual([f.code for f in findings], ["MULTIPLE"])
|
|
self.assertEqual(v, 1)
|
|
|
|
def test_empty_check_line(self):
|
|
findings, v = self.verdict(entry("CHECK:\n"))
|
|
self.assertEqual([f.code for f in findings], ["EMPTY"])
|
|
self.assertEqual(v, 1)
|
|
|
|
def test_check_inside_fence_is_not_the_contract(self):
|
|
text = entry("```\nCHECK: true\n```\n")
|
|
self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"])
|
|
|
|
def test_prose_starting_with_check_colon_is_not_matched(self):
|
|
text = entry("Note: CHECK: this is prose, not a contract line\n")
|
|
self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"])
|
|
|
|
# -- fallback arms -----------------------------------------------------
|
|
def test_fallback_or_true(self):
|
|
self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || true\n")))
|
|
|
|
def test_fallback_or_exit_0(self):
|
|
self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || exit 0\n")))
|
|
|
|
def test_fallback_semicolon_exit_0(self):
|
|
self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py; exit 0\n")))
|
|
|
|
def test_fallback_or_echo(self):
|
|
self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || echo ok\n")))
|
|
|
|
def test_fallback_is_reported_as_forced_exit_not_as_a_metachar_only(self):
|
|
findings = self.verdict(entry("CHECK: python3 a.py || true\n"))[0]
|
|
self.assertTrue(any("cannot fail" in f.message for f in findings))
|
|
|
|
# -- tautologies ------------------------------------------------------
|
|
def test_tautology_true(self):
|
|
self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: true\n")))
|
|
|
|
def test_tautology_exit_0(self):
|
|
self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: exit 0\n")))
|
|
|
|
def test_tautology_echo(self):
|
|
self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: echo ok\n")))
|
|
|
|
# -- metacharacters ----------------------------------------------------
|
|
def test_metachar_and_chain(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py && python3 b.py\n")))
|
|
|
|
def test_metachar_pipe(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: cat f | grep x\n")))
|
|
|
|
def test_metachar_redirect(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py >/dev/null 2>&1\n")))
|
|
|
|
def test_metachar_command_substitution(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: python3 $(ls)\n")))
|
|
|
|
def test_metachar_backtick(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: python3 `ls`\n")))
|
|
|
|
def test_metachar_line_continuation(self):
|
|
self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py \\\n")))
|
|
|
|
# -- real corpus shapes, copied verbatim --------------------------------
|
|
def test_real_corpus_and_chain_shape_is_flagged(self):
|
|
self.assertIn("METACHAR", self.codes(entry(
|
|
"CHECK: test -s tools/ok_gate.py && grep -q VERDICT tools/ok_gate.py\n")))
|
|
|
|
def test_real_corpus_redirect_shape_is_flagged(self):
|
|
self.assertIn("METACHAR", self.codes(entry(
|
|
"CHECK: python3 /root/troupe-standup/wizard-work/forge/verify.py >/dev/null 2>&1\n")))
|
|
|
|
# -- purity ------------------------------------------------------------
|
|
def test_lint_text_is_pure_and_never_executes(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
sentinel = Path(td) / "ran"
|
|
text = entry(f"CHECK: touch {sentinel}\n")
|
|
first = checklint.lint_text(text)
|
|
second = checklint.lint_text(text)
|
|
self.assertEqual(first, second) # pure: same input, same findings
|
|
self.assertEqual(first, []) # statically clean
|
|
self.assertFalse(sentinel.exists()) # and it did not execute anything
|
|
|
|
|
|
class CliTests(unittest.TestCase):
|
|
def run_tool(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, str(TOOL), *args],
|
|
capture_output=True, text=True, cwd=str(cwd) if cwd else None, timeout=120,
|
|
)
|
|
|
|
def test_cli_clean_entry_exits_0(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry(CLEAN), encoding="utf-8")
|
|
r = self.run_tool(str(p))
|
|
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
|
self.assertIn("OK", r.stdout)
|
|
|
|
def test_cli_faulty_entry_exits_1(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8")
|
|
r = self.run_tool(str(p))
|
|
self.assertEqual(r.returncode, 1, r.stdout + r.stderr)
|
|
self.assertIn("FALLBACK_ARM", r.stdout)
|
|
|
|
def test_cli_unreadable_entry_exits_2(self):
|
|
r = self.run_tool("/root/definitely/not/here.md")
|
|
self.assertEqual(r.returncode, 2)
|
|
self.assertIn("UNREADABLE", r.stderr)
|
|
|
|
def test_cli_run_leg_executes_the_command(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
sentinel = Path(td) / "ran"
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry(f"CHECK: touch {sentinel}\n"), encoding="utf-8")
|
|
r = self.run_tool("--run", str(p))
|
|
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
|
self.assertTrue(sentinel.exists(), "the CHECK command was not executed")
|
|
|
|
def test_cli_run_leg_reports_a_failing_command(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry("CHECK: false\n"), encoding="utf-8")
|
|
r = self.run_tool("--run", str(p))
|
|
self.assertEqual(r.returncode, 1)
|
|
self.assertIn("RUN_FAILURE", r.stdout)
|
|
self.assertIn("exit 1", r.stdout)
|
|
|
|
def test_cli_run_leg_does_not_execute_a_fallback_arm(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
sentinel = Path(td) / "ran"
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry(f"CHECK: touch {sentinel} || true\n"), encoding="utf-8")
|
|
r = self.run_tool("--run", str(p))
|
|
self.assertEqual(r.returncode, 1)
|
|
self.assertFalse(sentinel.exists())
|
|
|
|
def test_cli_timeout_exits_2_and_names_its_cause(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry("CHECK: sleep 30\n"), encoding="utf-8")
|
|
r = self.run_tool("--run", "--timeout", "0.5", str(p))
|
|
self.assertEqual(r.returncode, 2, r.stdout + r.stderr)
|
|
self.assertIn("TIMEOUT", r.stdout)
|
|
|
|
def test_cli_json_shape(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "e.md"
|
|
p.write_text(entry("CHECK: true\n"), encoding="utf-8")
|
|
r = self.run_tool("--json", str(p))
|
|
self.assertEqual(r.returncode, 1)
|
|
data = json.loads(r.stdout)
|
|
self.assertEqual(data["verdict"], 1)
|
|
self.assertEqual(data["findings"][0]["code"], "TAUTOLOGY")
|
|
self.assertIn("line", data["findings"][0])
|
|
|
|
def test_cli_no_entry_exits_2(self):
|
|
r = self.run_tool()
|
|
self.assertEqual(r.returncode, 2)
|
|
|
|
|
|
class ScanTests(unittest.TestCase):
|
|
def test_scan_gates_a_corpus(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root = Path(td)
|
|
(root / "clean.md").write_text(entry(CLEAN), encoding="utf-8")
|
|
(root / "faulty.md").write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8")
|
|
(root / "no-check.md").write_text("# nothing\n", encoding="utf-8")
|
|
(root / "bin").write_bytes(b"\x00\x01\x02")
|
|
r = subprocess.run(
|
|
[sys.executable, str(TOOL), "--scan", str(root)],
|
|
capture_output=True, text=True, timeout=120,
|
|
)
|
|
self.assertEqual(r.returncode, 1, r.stdout + r.stderr)
|
|
self.assertIn("2 file(s) with a CHECK line, 1 clean, 1 faulty", r.stdout)
|
|
self.assertIn("1 unreadable/binary skipped", r.stdout)
|
|
|
|
def test_scan_clean_tree_exits_0(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root = Path(td)
|
|
(root / "clean.md").write_text(entry(CLEAN), encoding="utf-8")
|
|
r = subprocess.run(
|
|
[sys.executable, str(TOOL), "--scan", str(root)],
|
|
capture_output=True, text=True, timeout=120,
|
|
)
|
|
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
|
|
|
def test_scan_unreadable_root_exits_2(self):
|
|
r = subprocess.run(
|
|
[sys.executable, str(TOOL), "--scan", "/root/definitely/not/a/dir"],
|
|
capture_output=True, text=True, timeout=120,
|
|
)
|
|
self.assertEqual(r.returncode, 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|