q100: the guard's test-file class is three shapes, not one -- TEST_NAME now reads _test_*.py and *_test.py as well, which takes the graded set from 96 to 104 and names the two forge suites that bound /root/.hermes/wizard-outputs/forge at _test_roundtrip.py:13 and _test_roundtrip_refactored.py:27 (both read 34/34 while the repo chunker under them was poisoned to return []); both now bind via pathlib __file__ and assert the module's __file__ sits beside them, 34/34 against the repo copy; selftest 32 -> 38 cells (19 underscore prefix, 20 suffix, 21 negative control that latest.py/attestation.py/protest.py stay advisory) and the baseline re-pinned 32 -> 38, regenerated with --write-baseline and diff-empty
This commit is contained in:
1 parent
41f4b370bd
commit
ed14f0ebc8
4 files changed
+85
-11
No files matched your search
@@ -27,13 +27,27 @@ WHY THIS EXISTS
|
|||||||
bind outside the tree.
|
bind outside the tree.
|
||||||
|
|
||||||
WHAT IT MEASURES -- a byte reading, not a text one
|
WHAT IT MEASURES -- a byte reading, not a text one
|
||||||
The scan parses each tracked `test_*.py` with `ast` and grades CALLS and
|
The scan parses each tracked test file with `ast` and grades CALLS and
|
||||||
ASSIGNMENTS, never prose. That distinction is the whole reason it is not a
|
ASSIGNMENTS, never prose. That distinction is the whole reason it is not a
|
||||||
`grep`: two of the nine hits are docstrings that RECORD the Q61 fix
|
`grep`: two of the nine hits are docstrings that RECORD the Q61 fix
|
||||||
(`wizard-work/beam/test_toposort.py:12`, `test_robin_hood_hash.py:5`), and a
|
(`wizard-work/beam/test_toposort.py:12`, `test_robin_hood_hash.py:5`), and a
|
||||||
grep-based scan reports them as faults forever -- which is how a guard gets
|
grep-based scan reports them as faults forever -- which is how a guard gets
|
||||||
switched off. `ast` also cannot be fooled by a commented-out insert.
|
switched off. `ast` also cannot be fooled by a commented-out insert.
|
||||||
|
|
||||||
|
WHAT COUNTS AS A TEST FILE -- three names, not one (Q100)
|
||||||
|
`test_*.py`, `_test_*.py`, `*_test.py`. The first draft of this guard read
|
||||||
|
only `test_*.py`, and Q100 measured what that cost: two tracked forge files,
|
||||||
|
`wizard-work/forge/_test_roundtrip.py:13` and
|
||||||
|
`_test_roundtrip_refactored.py:27`, each bound
|
||||||
|
`/root/.hermes/wizard-outputs/forge` and read **34 passed, 0 failed** while
|
||||||
|
the repo copy of `chunker.py` under them was poisoned to `return []`. A test
|
||||||
|
suite is a file that runs assertions and imports the module under test; the
|
||||||
|
spelling of its name is not evidence about what it does, so the class is
|
||||||
|
closed on the two shapes the first pass missed. A name that merely CONTAINS
|
||||||
|
`test` (`latest.py`, `attestation.py`, `protest.py`) is not a test file and
|
||||||
|
stays an ADVISORY row -- cell 21 pins that.
|
||||||
|
|
||||||
|
|
||||||
OUTSIDE_REPO `sys.path.insert/append/extend`, a `sys.path = [...]`
|
OUTSIDE_REPO `sys.path.insert/append/extend`, a `sys.path = [...]`
|
||||||
assignment, or `importlib.util.spec_from_file_location`
|
assignment, or `importlib.util.spec_from_file_location`
|
||||||
carries a constant absolute path that resolves OUTSIDE the
|
carries a constant absolute path that resolves OUTSIDE the
|
||||||
@@ -48,8 +62,9 @@ WHAT IT MEASURES -- a byte reading, not a text one
|
|||||||
UNGRADABLE a tracked test file that cannot be decoded as UTF-8 or
|
UNGRADABLE a tracked test file that cannot be decoded as UTF-8 or
|
||||||
parsed as Python. Not a pass: rc 2, because a file this
|
parsed as Python. Not a pass: rc 2, because a file this
|
||||||
instrument could not read is a file it did not grade.
|
instrument could not read is a file it did not grade.
|
||||||
ADVISORY an absolute binding in a tracked file that is not
|
ADVISORY an absolute binding in a tracked file that is not a test
|
||||||
`test_*.py`. Counted and printed, NOT graded -- the class
|
file by the name rule (`test_*.py`, `_test_*.py`,
|
||||||
|
`*_test.py`). Counted and printed, NOT graded -- the class
|
||||||
is 30 probe scripts across other lanes, most of them
|
is 30 probe scripts across other lanes, most of them
|
||||||
throwaway reproductions that name a host path on purpose,
|
throwaway reproductions that name a host path on purpose,
|
||||||
and reddening on them would make this guard unrunnable.
|
and reddening on them would make this guard unrunnable.
|
||||||
@@ -104,7 +119,7 @@ USAGE
|
|||||||
python3 import_resolve_check.py --all-files
|
python3 import_resolve_check.py --all-files
|
||||||
python3 import_resolve_check.py --import-probe
|
python3 import_resolve_check.py --import-probe
|
||||||
python3 import_resolve_check.py --json PATH
|
python3 import_resolve_check.py --json PATH
|
||||||
python3 import_resolve_check.py --selftest # 32 cells
|
python3 import_resolve_check.py --selftest # 38 cells
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -130,7 +145,7 @@ FATAL_CLASSES = (OUTSIDE_REPO, ABS_PATH_IN_REPO)
|
|||||||
# in a clone grades the clone, not a fixed path on the host that wrote it.
|
# in a clone grades the clone, not a fixed path on the host that wrote it.
|
||||||
DEFAULT_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
DEFAULT_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
TEST_NAME = re.compile(r"(^|/)test_[^/]*\.py$")
|
TEST_NAME = re.compile(r"(^|/)(_?test_[^/]*|[^/]*_test)\.py$")
|
||||||
TRACEBACK = "Traceback (most recent call last)"
|
TRACEBACK = "Traceback (most recent call last)"
|
||||||
|
|
||||||
# sys.path mutators whose last argument is the path being added.
|
# sys.path mutators whose last argument is the path being added.
|
||||||
@@ -799,6 +814,51 @@ def selftest():
|
|||||||
"--quiet"])
|
"--quiet"])
|
||||||
check("18c an unwritable --json target -> rc 2", rc2, 2)
|
check("18c an unwritable --json target -> rc 2", rc2, 2)
|
||||||
|
|
||||||
|
# 19. Q100: an UNDERSCORE-prefixed suite is a test file. This is the
|
||||||
|
# measured shape: both forge `_test_roundtrip*.py` files bound
|
||||||
|
# /root/.hermes/wizard-outputs/forge and read 34/34 while the repo copy
|
||||||
|
# under them was poisoned. Under the first selector they were invisible.
|
||||||
|
repo = os.path.join(td, "c19")
|
||||||
|
_write(os.path.join(repo, "_test_roundtrip.py"),
|
||||||
|
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/forge")\n'
|
||||||
|
'import chunker\n')
|
||||||
|
_write(os.path.join(repo, "chunker.py"), "def chunk_text(t, **k):\n return []\n")
|
||||||
|
_git_init(repo)
|
||||||
|
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||||
|
capture_output=True)
|
||||||
|
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||||
|
check("19 an _test_*.py host binding -> rc 1", rc, 1)
|
||||||
|
check("19b and it is graded, not advised",
|
||||||
|
(obj["graded"], obj["faults"][0][0], obj["faults"][0][1]),
|
||||||
|
(1, "_test_roundtrip.py", 2))
|
||||||
|
|
||||||
|
# 20. and the suffix spelling binds the same way
|
||||||
|
repo = os.path.join(td, "c20")
|
||||||
|
_write(os.path.join(repo, "roundtrip_test.py"),
|
||||||
|
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/forge")\n')
|
||||||
|
_git_init(repo)
|
||||||
|
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||||
|
capture_output=True)
|
||||||
|
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||||
|
check("20 a *_test.py host binding -> rc 1", rc, 1)
|
||||||
|
check("20b named as a fault row", obj["faults"][0][0],
|
||||||
|
"roundtrip_test.py")
|
||||||
|
|
||||||
|
# 21. NEGATIVE CONTROL: a name that merely CONTAINS `test` is not a
|
||||||
|
# test file. Widening the class must not sweep in latest.py/protest.py
|
||||||
|
# and turn the advisory count into a gate nobody can run.
|
||||||
|
repo = os.path.join(td, "c21")
|
||||||
|
for name in ("latest.py", "attestation.py", "protest.py"):
|
||||||
|
_write(os.path.join(repo, name),
|
||||||
|
'import sys\nsys.path.insert(0, "/root/.hermes/wizard-outputs/x")\n')
|
||||||
|
_git_init(repo)
|
||||||
|
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
|
||||||
|
capture_output=True)
|
||||||
|
rc, obj = call(repo, "WORKTREE", False, False, quiet=True)
|
||||||
|
check("21 a name containing `test` is not a test file -> rc 0", rc, 0)
|
||||||
|
check("21b it stays an advisory row, ungraded",
|
||||||
|
(obj["graded"], obj["advisory"]), (0, 3))
|
||||||
|
|
||||||
print("selftest: %d cells, %d passed, %d failed"
|
print("selftest: %d cells, %d passed, %d failed"
|
||||||
% (cells, passed, len(failures)))
|
% (cells, passed, len(failures)))
|
||||||
for f in failures:
|
for f in failures:
|
||||||
@@ -810,8 +870,9 @@ def selftest():
|
|||||||
|
|
||||||
def main(argv=None):
|
def main(argv=None):
|
||||||
ap = argparse.ArgumentParser(
|
ap = argparse.ArgumentParser(
|
||||||
description="Grade every committed test_*.py on where it binds the "
|
description="Grade every committed test file (test_*.py, _test_*.py, "
|
||||||
"module it imports: the repo, or a path on the host.")
|
"*_test.py) on where it binds the module it imports: the "
|
||||||
|
"repo, or a path on the host.")
|
||||||
ap.add_argument("--repo", default=DEFAULT_REPO,
|
ap.add_argument("--repo", default=DEFAULT_REPO,
|
||||||
help="git work tree root (default: the checkout holding "
|
help="git work tree root (default: the checkout holding "
|
||||||
"this tool)")
|
"this tool)")
|
||||||
@@ -819,7 +880,7 @@ def main(argv=None):
|
|||||||
help="WORKTREE (the index, default) or a committed "
|
help="WORKTREE (the index, default) or a committed "
|
||||||
"revision such as HEAD")
|
"revision such as HEAD")
|
||||||
ap.add_argument("--all-files", action="store_true",
|
ap.add_argument("--all-files", action="store_true",
|
||||||
help="grade every tracked *.py, not only test_*.py -- the "
|
help="grade every tracked *.py, not only test files -- the "
|
||||||
"non-test class is advisory by default")
|
"non-test class is advisory by default")
|
||||||
ap.add_argument("--import-probe", action="store_true",
|
ap.add_argument("--import-probe", action="store_true",
|
||||||
help="also load each test file in a clean subprocess and "
|
help="also load each test file in a clean subprocess and "
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ citation_resolve_check.py 14 fixtures
|
|||||||
doc_census_check.py 104 cells
|
doc_census_check.py 104 cells
|
||||||
doc_command_check.py 18 cells
|
doc_command_check.py 18 cells
|
||||||
gate_blame.py 21 cells
|
gate_blame.py 21 cells
|
||||||
import_resolve_check.py 32 cells
|
import_resolve_check.py 38 cells
|
||||||
landing_check.py 26 cells
|
landing_check.py 26 cells
|
||||||
manifest_audit.py 17 cells
|
manifest_audit.py 17 cells
|
||||||
path_claim_check.py 31 cells
|
path_claim_check.py 31 cells
|
||||||
|
|||||||
|
@@ -10,9 +10,16 @@ Covers:
|
|||||||
import sys
|
import sys
|
||||||
import random
|
import random
|
||||||
import string
|
import string
|
||||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
import pathlib
|
||||||
|
HERE = pathlib.Path(__file__).resolve().parent
|
||||||
|
sys.path.insert(0, str(HERE))
|
||||||
import chunker
|
import chunker
|
||||||
|
|
||||||
|
# Q100: grade the file this suite sits beside, not a copy on the host that
|
||||||
|
# happened to write it. Without this line the suite read 34/34 while the repo
|
||||||
|
# copy under it was poisoned to `return []`.
|
||||||
|
assert pathlib.Path(chunker.__file__).resolve().parent == HERE, chunker.__file__
|
||||||
|
|
||||||
PASS = 0
|
PASS = 0
|
||||||
FAIL = 0
|
FAIL = 0
|
||||||
|
|
||||||
|
|||||||
@@ -23,11 +23,17 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import random
|
import random
|
||||||
import sys
|
import sys
|
||||||
|
import pathlib
|
||||||
|
|
||||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
HERE = pathlib.Path(__file__).resolve().parent
|
||||||
|
sys.path.insert(0, str(HERE))
|
||||||
|
|
||||||
import chunker
|
import chunker
|
||||||
|
|
||||||
|
# Q100: this suite grades the `chunker.py` committed beside it, not the host
|
||||||
|
# copy at /root/.hermes/wizard-outputs/forge that the first line bound.
|
||||||
|
assert pathlib.Path(chunker.__file__).resolve().parent == HERE, chunker.__file__
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# diagnostics
|
# diagnostics
|
||||||
|
|||||||
Reference in new issue
Block a user