volt
ecec64b95b
test_shell_lone_copy_reds_on_its_sibling asserted the fixture path it had just built while verify_hermes_setup.sh reports the path it resolved with readlink -f. Under TMPDIR=/root/.hermes/profiles/volt/cache/scratch_link a correct pair read MISSING and the gate went rc 1 (measured before the repair: that cell was the only red, 15 others ok). The fixture root is now canonicalised once in setUp with os.path.realpath; two new cells pin the alias reading instead of assuming it -- a lone copy reached by a directory symlink names the real sibling that is absent (rc 1), and the same alias with the sibling present reads PASS (rc 0). CHECK-env-verifiers now runs the suite twice, ambient TMPDIR and a directory alias it builds, so the shape that produced the flake is graded every run. 16 cells; negative control (raw mkdtemp restored in a scratch copy) reds the alias leg and only that leg, rc 1.
282 lines
14 KiB
Python
Executable File
282 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Q109 cells: the lane's two environment verifiers, graded as subprocesses.
|
|
|
|
Both files used to print success without measuring anything --
|
|
`wizard-work/spark/snippet.py` filled `available[tool] = True` in a loop over a
|
|
hard-coded list (so a bare python3 with no Hermes session read "verified
|
|
successfully", rc 0), and `wizard-work/spark/verify_hermes_setup.sh` tested
|
|
three absolute paths under a pinned /root and read nothing relative to itself
|
|
(so it printed three ticks and "All critical paths verified", rc 0, on any
|
|
tree). Those are the two readings these cells kill.
|
|
|
|
Every cell runs the shipped file by path, resolved from __file__, and imports
|
|
nothing from it: no cell can agree with the code by sharing its constants. The
|
|
tool check is graded with a stub `hermes_tools` on PYTHONPATH -- full, reduced,
|
|
and non-callable -- and the directory check is graded against synthetic trees
|
|
built here, so a green reading is a statement about the arguments and not about
|
|
this host.
|
|
|
|
Two cells grade the alias case, and they exist because this file was the thing
|
|
that broke, not the code under it. The first version of the lone-copy cell
|
|
compared the script's reported self-location (which it resolves with
|
|
`readlink -f`) against the fixture path this cell had just built. Under a TMPDIR
|
|
reached through a symlink those differ and the cell read a false MISSING on a
|
|
correct pair -- the gate's only red that day. The fixture root is now
|
|
canonicalised once in setUp, and `test_shell_lone_copy_behind_a_symlinked_dir`
|
|
and `test_shell_pair_behind_a_symlinked_dir_is_green` pin the reading the script
|
|
must give when it is reached by an alias.
|
|
|
|
Run: python3 test_env_verifiers.py (or: python3 -m unittest -q ...)
|
|
"""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
LANE = Path(__file__).resolve().parent
|
|
SNIPPET = LANE / "snippet.py"
|
|
SHELL = LANE / "verify_hermes_setup.sh"
|
|
|
|
TOOLS = ["web_search", "web_extract", "read_file", "write_file"]
|
|
|
|
STUB_HEADER = '''"""Stub hermes_tools written by test_env_verifiers.py."""\n'''
|
|
|
|
|
|
def stub_source(names):
|
|
"""A stand-in hermes_tools exposing `names` as callables."""
|
|
lines = [STUB_HEADER]
|
|
for name in names:
|
|
lines.append("def %s(*args, **kwargs):\n return None\n" % name)
|
|
return "".join(lines)
|
|
|
|
|
|
class EnvVerifiers(unittest.TestCase):
|
|
def setUp(self):
|
|
# realpath, not the raw mkdtemp: this process may itself be running under
|
|
# a TMPDIR whose path goes through a symlink (macOS /var -> /private/var,
|
|
# a linked cache dir), and `verify_hermes_setup.sh` resolves its own
|
|
# self-location with `readlink -f`. Comparing its resolved path against a
|
|
# fixture path that still carries the alias reads a false MISSING -- the
|
|
# flake that reds `test_shell_lone_copy_reds_on_its_sibling` on a host
|
|
# where /tmp is not canonical. One canonical root for every fixture.
|
|
self.tmp = Path(os.path.realpath(
|
|
tempfile.mkdtemp(prefix="q109-cell-", dir=tempfile.gettempdir())))
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
|
|
# ---- fixtures -------------------------------------------------------
|
|
|
|
def make_tree(self, name="tree", scratch=True, home=True):
|
|
"""A synthetic Hermes installation: ROOT/profiles/<profile>/cache/scratch."""
|
|
root = self.tmp / name
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / "skills").mkdir(exist_ok=True)
|
|
(root / "plugins").mkdir(exist_ok=True)
|
|
profile_home = root / "profiles" / "lane"
|
|
if home:
|
|
profile_home.mkdir(parents=True, exist_ok=True)
|
|
if scratch and home:
|
|
(profile_home / "cache" / "scratch").mkdir(parents=True, exist_ok=True)
|
|
return root, profile_home
|
|
|
|
def make_stub(self, names, label="stub"):
|
|
"""A PYTHONPATH directory holding a hermes_tools stub, or none at all."""
|
|
stub_dir = self.tmp / label
|
|
stub_dir.mkdir(parents=True, exist_ok=True)
|
|
if names is not None:
|
|
(stub_dir / "hermes_tools.py").write_text(stub_source(names), encoding="utf-8")
|
|
return stub_dir
|
|
|
|
def env_for(self, stub_dir, **extra):
|
|
env = {k: v for k, v in os.environ.items()
|
|
if k not in ("HERMES_HOME", "HERMES_ROOT", "HERMES_PROFILE", "PYTHONPATH")}
|
|
env["PYTHONPATH"] = str(stub_dir)
|
|
env.update(extra)
|
|
return env
|
|
|
|
def run_snippet(self, stub_dir, *args, **extra_env):
|
|
return subprocess.run(
|
|
[sys.executable, str(SNIPPET)] + [str(a) for a in args],
|
|
capture_output=True, text=True, env=self.env_for(stub_dir, **extra_env),
|
|
)
|
|
|
|
def run_shell(self, script, *args, **extra_env):
|
|
stub_dir = self.make_stub(None, label="no-stub")
|
|
return subprocess.run(
|
|
["bash", str(script)] + [str(a) for a in args],
|
|
capture_output=True, text=True, env=self.env_for(stub_dir, **extra_env),
|
|
)
|
|
|
|
def lines(self, result, tag):
|
|
return [ln[len(tag) + 2:] for ln in result.stdout.splitlines() if ln.startswith(tag + ": ")]
|
|
|
|
def named(self, result, path):
|
|
return any(path in ln for ln in result.stdout.splitlines())
|
|
|
|
# ---- snippet.py: the tool check measures ----------------------------
|
|
|
|
def test_snippet_bare_interpreter_reds_on_tools(self):
|
|
"""The defect itself: no hermes_tools at all must not read as verified."""
|
|
root, home = self.make_tree()
|
|
result = self.run_snippet(self.make_stub(None, "empty"), "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertIn("TOOLS_IMPORT_FAILED:", result.stdout)
|
|
self.assertEqual(self.lines(result, "TOOLS_MISSING"), [str(TOOLS)])
|
|
self.assertIn("VERDICT: FAIL", result.stdout)
|
|
|
|
def test_snippet_full_stub_is_green(self):
|
|
"""Positive control: every tool callable on a complete tree reads PASS."""
|
|
root, home = self.make_tree("green")
|
|
result = self.run_snippet(self.make_stub(TOOLS), "--home", home)
|
|
self.assertEqual(result.returncode, 0, result.stdout)
|
|
self.assertEqual(self.lines(result, "TOOLS_AVAILABLE"), [str(sorted(TOOLS))])
|
|
self.assertEqual(self.lines(result, "TOOLS_MISSING"), ["[]"])
|
|
self.assertIn("VERDICT: PASS", result.stdout)
|
|
self.assertEqual(result.stderr, "")
|
|
|
|
def test_snippet_one_tool_missing_reds_and_names_it(self):
|
|
"""The negative control the item asks for: one genuinely absent tool."""
|
|
root, home = self.make_tree("partial")
|
|
stub = self.make_stub([t for t in TOOLS if t != "write_file"], "partial-stub")
|
|
result = self.run_snippet(stub, "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertEqual(self.lines(result, "TOOLS_MISSING"), ["['write_file']"])
|
|
|
|
def test_snippet_noncallable_tool_is_unavailable(self):
|
|
"""A name that resolves to None is not a tool that works."""
|
|
root, home = self.make_tree("noncallable")
|
|
stub_dir = self.make_stub(TOOLS, "noncallable-stub")
|
|
(stub_dir / "hermes_tools.py").write_text(
|
|
stub_source(TOOLS) + "web_extract = None\n", encoding="utf-8")
|
|
result = self.run_snippet(stub_dir, "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertEqual(self.lines(result, "TOOLS_MISSING"), ["['web_extract']"])
|
|
|
|
def test_snippet_missing_scratch_dir_reds_and_names_path(self):
|
|
root, home = self.make_tree("noscratch", scratch=False)
|
|
result = self.run_snippet(self.make_stub(TOOLS), "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertTrue(self.named(result, str(home / "cache" / "scratch")), result.stdout)
|
|
self.assertIn("DIR MISSING: %s" % (home / "cache" / "scratch"), result.stdout)
|
|
|
|
def test_snippet_require_absent_tool_reds(self):
|
|
root, home = self.make_tree("require")
|
|
result = self.run_snippet(self.make_stub(TOOLS), "--home", home,
|
|
"--require", "definitely_not_a_tool")
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertEqual(self.lines(result, "TOOLS_MISSING"), ["['definitely_not_a_tool']"])
|
|
|
|
def test_snippet_honours_hermes_home_env(self):
|
|
"""Resolution is env-first: HERMES_HOME alone must redirect the reading."""
|
|
root, home = self.make_tree("envhome")
|
|
result = self.run_snippet(self.make_stub(TOOLS), HERMES_HOME=str(home))
|
|
self.assertEqual(result.returncode, 0, result.stdout)
|
|
self.assertEqual(self.lines(result, "HERMES_HOME"), [str(home)])
|
|
self.assertEqual(self.lines(result, "HERMES_ROOT"), [str(root)])
|
|
|
|
def test_snippet_usage_error_is_2(self):
|
|
root, home = self.make_tree("usage")
|
|
result = self.run_snippet(self.make_stub(TOOLS), "--nope")
|
|
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
|
|
|
|
# ---- verify_hermes_setup.sh: reads its own inputs -------------------
|
|
|
|
def copy_pair(self, name="copy"):
|
|
"""The two files as they ship together, outside the repo."""
|
|
target = self.tmp / name
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(SNIPPET, target / SNIPPET.name)
|
|
shutil.copy2(SHELL, target / SHELL.name)
|
|
return target
|
|
|
|
def test_shell_complete_tree_is_green(self):
|
|
root, home = self.make_tree("shell-green")
|
|
script = self.copy_pair() / SHELL.name
|
|
result = self.run_shell(script, "--home", home)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertTrue(self.named(result, str(home)), result.stdout)
|
|
self.assertIn("VERDICT: PASS", result.stdout)
|
|
|
|
def test_shell_missing_profile_home_reds_and_names_path(self):
|
|
root, home = self.make_tree("shell-red", home=False)
|
|
result = self.run_shell(self.copy_pair("copy-red") / SHELL.name, "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertIn("MISSING: %s" % home, result.stdout)
|
|
self.assertIn("VERDICT: FAIL (2 required path(s) missing)", result.stdout)
|
|
|
|
def test_shell_lone_copy_reds_on_its_sibling(self):
|
|
"""The self-location leg: a script with no snippet.py beside it has nothing to report."""
|
|
lone = self.tmp / "lone"
|
|
lone.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(SHELL, lone / SHELL.name)
|
|
root, home = self.make_tree("shell-lone")
|
|
result = self.run_shell(lone / SHELL.name, "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertIn("MISSING: %s" % (lone / "snippet.py"), result.stdout)
|
|
|
|
def test_shell_lone_copy_behind_a_symlinked_dir(self):
|
|
"""Reached by an alias, a lone copy still names the sibling that is not there.
|
|
|
|
The alias is a directory symlink the cell builds, so this is not a
|
|
statement about /tmp on this host: whatever path the script was invoked
|
|
by, the path it reports must be the ones the kernel would resolve -- the
|
|
real directory, where the sibling genuinely is absent.
|
|
"""
|
|
real = self.tmp / "real-lone"
|
|
real.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(SHELL, real / SHELL.name)
|
|
alias = self.tmp / "alias-lone"
|
|
alias.symlink_to(real, target_is_directory=True)
|
|
root, home = self.make_tree("shell-alias-lone")
|
|
result = self.run_shell(alias / SHELL.name, "--home", home)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertIn("script dir: %s" % real, result.stdout)
|
|
self.assertIn("MISSING: %s" % (real / "snippet.py"), result.stdout)
|
|
self.assertTrue(os.path.exists(alias / SHELL.name))
|
|
self.assertFalse(os.path.exists(real / "snippet.py"))
|
|
|
|
def test_shell_pair_behind_a_symlinked_dir_is_green(self):
|
|
"""The same alias with the sibling present: the reading follows the file, not the name."""
|
|
real = self.copy_pair("real-pair")
|
|
alias = self.tmp / "alias-pair"
|
|
alias.symlink_to(real, target_is_directory=True)
|
|
root, home = self.make_tree("shell-alias-green")
|
|
result = self.run_shell(alias / SHELL.name, "--home", home)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertIn("script dir: %s" % real, result.stdout)
|
|
self.assertIn("ok: %s" % (real / SNIPPET.name), result.stdout)
|
|
self.assertIn("VERDICT: PASS", result.stdout)
|
|
|
|
def test_shell_same_verdict_from_outside_copy(self):
|
|
"""In place and outside the repo, on the same inputs: the same reading."""
|
|
root, home = self.make_tree("shell-same")
|
|
in_place = self.run_shell(SHELL, "--home", home)
|
|
outside = self.run_shell(self.copy_pair("copy-same") / SHELL.name, "--home", home)
|
|
self.assertEqual(in_place.returncode, outside.returncode)
|
|
self.assertEqual(in_place.returncode, 0, in_place.stdout + outside.stdout)
|
|
# Every line about the graded tree, not about where the script lives.
|
|
about_tree = lambda out: [ln for ln in out.splitlines() if str(root) in ln]
|
|
self.assertEqual(about_tree(in_place.stdout), about_tree(outside.stdout))
|
|
self.assertIn("VERDICT: PASS", outside.stdout)
|
|
|
|
def test_shell_usage_error_is_2(self):
|
|
result = self.run_shell(SHELL, "--nope")
|
|
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
|
|
|
|
# ---- the class both files were filed under -------------------------
|
|
|
|
def test_no_pinned_root_paths_in_either_file(self):
|
|
"""Q109's class: neither file may grade a path pinned to this host."""
|
|
for path in (SNIPPET, SHELL):
|
|
blob = path.read_text(encoding="utf-8")
|
|
self.assertNotIn("/root", blob, "%s still pins a /root path" % path.name)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|