q100: the guard's test-file class is three shapes, not one -- TEST_NAME now reads _test_*.py and *_test.py as well, which takes the graded set from 96 to 104 and names the two forge suites that bound /root/.hermes/wizard-outputs/forge at _test_roundtrip.py:13 and _test_roundtrip_refactored.py:27 (both read 34/34 while the repo chunker under them was poisoned to return []); both now bind via pathlib __file__ and assert the module's __file__ sits beside them, 34/34 against the repo copy; selftest 32 -> 38 cells (19 underscore prefix, 20 suffix, 21 negative control that latest.py/attestation.py/protest.py stay advisory) and the baseline re-pinned 32 -> 38, regenerated with --write-baseline and diff-empty
This commit is contained in:
1 parent
41f4b370bd
commit
ed14f0ebc8
4 files changed
+85
-11
No files matched your search
@@ -27,13 +27,27 @@ WHY THIS EXISTS
|
||||
bind outside the tree.
|
||||
|
||||
WHAT IT MEASURES -- a byte reading, not a text one
|
||||
The scan parses each tracked `test_*.py` with `ast` and grades CALLS and
|
||||
The scan parses each tracked test file with `ast` and grades CALLS and
|
||||
ASSIGNMENTS, never prose. That distinction is the whole reason it is not a
|
||||
`grep`: two of the nine hits are docstrings that RECORD the Q61 fix
|
||||
(`wizard-work/beam/test_toposort.py:12`, `test_robin_hood_hash.py:5`), and a
|
||||
grep-based scan reports them as faults forever -- which is how a guard gets
|
||||
switched off. `ast` also cannot be fooled by a commented-out insert.
|
||||
|
||||
WHAT COUNTS AS A TEST FILE -- three names, not one (Q100)
|
||||
`test_*.py`, `_test_*.py`, `*_test.py`. The first draft of this guard read
|
||||
only `test_*.py`, and Q100 measured what that cost: two tracked forge files,
|
||||
`wizard-work/forge/_test_roundtrip.py:13` and
|
||||
`_test_roundtrip_refactored.py:27`, each bound
|
||||
`/root/.hermes/wizard-outputs/forge` and read **34 passed, 0 failed** while
|
||||
the repo copy of `chunker.py` under them was poisoned to `return []`. A test
|
||||
suite is a file that runs assertions and imports the module under test; the
|
||||
spelling of its name is not evidence about what it does, so the class is
|
||||
closed on the two shapes the first pass missed. A name that merely CONTAINS
|
||||
`test` (`latest.py`, `attestation.py`, `protest.py`) is not a test file and
|
||||
stays an ADVISORY row -- cell 21 pins that.
|
||||
|
||||
|
||||
OUTSIDE_REPO `sys.path.insert/append/extend`, a `sys.path = [...]`
|
||||
assignment, or `importlib.util.spec_from_file_location`
|
||||
carries a constant absolute path that resolves OUTSIDE the
|
||||
@@ -48,8 +62,9 @@ WHAT IT MEASURES -- a byte reading, not a text one
|
||||
UNGRADABLE a tracked test file that cannot be decoded as UTF-8 or
|
||||
parsed as Python. Not a pass: rc 2, because a file this
|
||||
instrument could not read is a file it did not grade.
|
||||
ADVISORY an absolute binding in a tracked file that is not
|
||||
`test_*.py`. Counted and printed, NOT graded -- the class
|
||||
ADVISORY an absolute binding in a tracked file that is not a test
|
||||
file by the name rule (`test_*.py`, `_test_*.py`,
|
||||
`*_test.py`). Counted and printed, NOT graded -- the class
|
||||
is 30 probe scripts across other lanes, most of them
|
||||
throwaway reproductions that name a host path on purpose,
|
||||
and reddening on them would make this guard unrunnable.
|
||||
@@ -104,7 +119,7 @@ USAGE
|
||||
python3 import_resolve_check.py --all-files
|
||||
python3 import_resolve_check.py --import-probe
|
||||
python3 import_resolve_check.py --json PATH
|
||||
python3 import_resolve_check.py --selftest # 32 cells
|
||||
python3 import_resolve_check.py --selftest # 38 cells
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -130,7 +145,7 @@ FATAL_CLASSES = (OUTSIDE_REPO, ABS_PATH_IN_REPO)
|
||||
# in a clone grades the clone, not a fixed path on the host that wrote it.
|
||||
DEFAULT_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
TEST_NAME = re.compile(r"(^|/)test_[^/]*\.py$")
|
||||
TEST_NAME = re.compile(r"(^|/)(_?test_[^/]*|[^/]*_test)\.py$")
|
||||
TRACEBACK = "Traceback (most recent call last)"
|
||||
|
||||
# sys.path mutators whose last argument is the path being added.
|
||||
@@ -799,6 +814,51 @@ def selftest():
|
||||
"--quiet"])
|
||||
check("18c an unwritable --json target -> rc 2", rc2, 2)
|
||||
|
||||
# 19. Q100: an UNDERSCORE-prefixed suite is a test file. This is the
|
||||
# measured shape: both forge `_test_roundtrip*.py` files bound
|
||||
# /root/.hermes/wizard-outputs/forge and read 34/34 while the repo copy
|
||||
# under them was poisoned. Under the first selector they were invisible.
|
||||
repo = os.path.join(td, "c19")
|
||||
_write(os.path.join(repo, "_test_roundtrip.py"),
|
||||
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/forge")\n'
|
||||
'import chunker\n')
|
||||
_write(os.path.join(repo, "chunker.py"), "def chunk_text(t, **k):\n return []\n")
|
||||
_git_init(repo)
|
||||
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||
capture_output=True)
|
||||
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||
check("19 an _test_*.py host binding -> rc 1", rc, 1)
|
||||
check("19b and it is graded, not advised",
|
||||
(obj["graded"], obj["faults"][0][0], obj["faults"][0][1]),
|
||||
(1, "_test_roundtrip.py", 2))
|
||||
|
||||
# 20. and the suffix spelling binds the same way
|
||||
repo = os.path.join(td, "c20")
|
||||
_write(os.path.join(repo, "roundtrip_test.py"),
|
||||
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/forge")\n')
|
||||
_git_init(repo)
|
||||
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||
capture_output=True)
|
||||
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||
check("20 a *_test.py host binding -> rc 1", rc, 1)
|
||||
check("20b named as a fault row", obj["faults"][0][0],
|
||||
"roundtrip_test.py")
|
||||
|
||||
# 21. NEGATIVE CONTROL: a name that merely CONTAINS `test` is not a
|
||||
# test file. Widening the class must not sweep in latest.py/protest.py
|
||||
# and turn the advisory count into a gate nobody can run.
|
||||
repo = os.path.join(td, "c21")
|
||||
for name in ("latest.py", "attestation.py", "protest.py"):
|
||||
_write(os.path.join(repo, name),
|
||||
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/x")\n')
|
||||
_git_init(repo)
|
||||
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||
capture_output=True)
|
||||
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||
check("21 a name containing `test` is not a test file -> rc 0", rc, 0)
|
||||
check("21b it stays an advisory row, ungraded",
|
||||
(obj["graded"], obj["advisory"]), (0, 3))
|
||||
|
||||
print("selftest: %d cells, %d passed, %d failed"
|
||||
% (cells, passed, len(failures)))
|
||||
for f in failures:
|
||||
@@ -810,8 +870,9 @@ def selftest():
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Grade every committed test_*.py on where it binds the "
|
||||
"module it imports: the repo, or a path on the host.")
|
||||
description="Grade every committed test file (test_*.py, _test_*.py, "
|
||||
"*_test.py) on where it binds the module it imports: the "
|
||||
"repo, or a path on the host.")
|
||||
ap.add_argument("--repo", default=DEFAULT_REPO,
|
||||
help="git work tree root (default: the checkout holding "
|
||||
"this tool)")
|
||||
@@ -819,7 +880,7 @@ def main(argv=None):
|
||||
help="WORKTREE (the index, default) or a committed "
|
||||
"revision such as HEAD")
|
||||
ap.add_argument("--all-files", action="store_true",
|
||||
help="grade every tracked *.py, not only test_*.py -- the "
|
||||
help="grade every tracked *.py, not only test files -- the "
|
||||
"non-test class is advisory by default")
|
||||
ap.add_argument("--import-probe", action="store_true",
|
||||
help="also load each test file in a clean subprocess and "
|
||||
|
||||
@@ -19,7 +19,7 @@ citation_resolve_check.py 14 fixtures
|
||||
doc_census_check.py 104 cells
|
||||
doc_command_check.py 18 cells
|
||||
gate_blame.py 21 cells
|
||||
import_resolve_check.py 32 cells
|
||||
import_resolve_check.py 38 cells
|
||||
landing_check.py 26 cells
|
||||
manifest_audit.py 17 cells
|
||||
path_claim_check.py 31 cells
|
||||
|
||||
|
@@ -10,9 +10,16 @@ Covers:
|
||||
import sys
|
||||
import random
|
||||
import string
|
||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
||||
import pathlib
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import chunker
|
||||
|
||||
# Q100: grade the file this suite sits beside, not a copy on the host that
|
||||
# happened to write it. Without this line the suite read 34/34 while the repo
|
||||
# copy under it was poisoned to `return []`.
|
||||
assert pathlib.Path(chunker.__file__).resolve().parent == HERE, chunker.__file__
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
|
||||
@@ -23,11 +23,17 @@ from __future__ import annotations
|
||||
|
||||
import random
|
||||
import sys
|
||||
import pathlib
|
||||
|
||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import chunker
|
||||
|
||||
# Q100: this suite grades the `chunker.py` committed beside it, not the host
|
||||
# copy at /root/.hermes/wizard-outputs/forge that the first line bound.
|
||||
assert pathlib.Path(chunker.__file__).resolve().parent == HERE, chunker.__file__
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# diagnostics
|
||||
|
||||
Reference in new issue
Block a user