#!/usr/bin/env python3 """q69_hostroot_census.py -- classify every mention of a host-absolute path in the tracked `wizard-work/**/*.py` population by the position it occupies in the AST, so a docstring that RECORDS a defect is counted apart from a value the interpreter can use. WHY THIS EXISTS Q69 counts "committed files that name the absolute path /root/troupe-standup" with `git grep -l`, and Q62 counts the same shape against /root/.hermes/wizard-outputs: 116 and 168 at the commits those items name. A text census cannot tell a gate that READS the author's tree from a docstring that records where the defect used to be. Q63's rule -- prose about a defect is not wiring for it -- runs in both directions: prose is not an instance of the defect either. Measured at this tree, a text scan reports 586 of 1345 tracked `wizard-work/**/*.py` as naming /root/ or /home/; this census is the other number, the one a repair queue needs: how many of those name it in a position code can execute. WHAT IT MEASURES -- position, not text A host-absolute mention is an occurrence of `/root/`, `/home/`, `/var/`, `/srv/`, `/opt/` or `/mnt/` in a `str` constant. `/usr`, `/bin`, `/etc` and `/tmp` are excluded on purpose: they hold the machine's tools and scratch, not one author's checkout or state directory, and including them would drown the figure in shebangs and TemporaryDirectory prefixes. PATTERN a string equal to a bare host prefix (`"/root/"`, `"/home/"`, ...). That names a namespace, not a location: it is the shape a matcher compares against, so this census's own HOST_PREFIXES table is a PATTERN row and not six host-path values. Counted apart, never graded -- otherwise the instrument inflates the population it reports by the table it measures with. PROSE the constant is the whole of an expression statement -- a module, class or function docstring. Comments never reach the AST, so a commented-out binding is prose by construction. Reported, never counted as an instance. CODE any other position. Two buckets, because they need different treatment: a constant that STARTS with a host prefix is a path VALUE (graded, with a subtype below); a constant that merely CONTAINS a host prefix mid-string is an executable mention -- a printed message, a fixture, a selftest string -- and is counted but not graded. Subtypes for CODE rows: BIND sys.path.insert/append/extend, a `sys.path = [...]` element, or importlib.util.spec_from_file_location's path RUN an argv element or a `cwd=` of subprocess/os.system/os.popen, or a shutil source/target READ the path argument of open/Path/read_text/read_bytes, or the FIRST argument of os.path.join/exists/isfile/isdir, which makes the whole joined path absolute CONST a module constant, default argument, return value or other value: code the interpreter can use, but not by itself a read A CODE row is not automatically a fault. Q69's done-means allow a file to state that it deliberately grades the live tree; this census finds the files that need that statement and does not decide which ones already have it. THE EXIT CONTRACT -- the house three codes 0 every tracked file was read and parsed; the table is a complete reading 1 a --lane was named that holds no tracked file (nothing was graded) 2 the inventory, a file, or a --json target could not be read, or a file did not parse: a run that could not read part of its inventory has not earned a claim about the whole tree USAGE python3 wizard-work/volt/q69_hostroot_census.py python3 wizard-work/volt/q69_hostroot_census.py --lane wizard-work/volt python3 wizard-work/volt/q69_hostroot_census.py --list --json rows.json python3 wizard-work/volt/q69_hostroot_census.py --selftest # 12 cells """ from __future__ import annotations import argparse import ast import json import os import subprocess import sys import tokenize HOST_PREFIXES = ("/root/", "/home/", "/var/", "/srv/", "/opt/", "/mnt/") BIND_FUNCS = ("sys.path.insert", "sys.path.append", "sys.path.extend", "importlib.util.spec_from_file_location") RUN_FUNCS = ("subprocess.run", "subprocess.Popen", "subprocess.call", "subprocess.check_call", "subprocess.check_output", "os.system", "os.popen", "shutil.copy", "shutil.copyfile", "shutil.copytree") READ_FUNCS = ("open", "io.open", "Path", "pathlib.Path", "os.path.join", "os.path.exists", "os.path.isfile", "os.path.isdir", "os.listdir", "os.walk", "os.chdir") READ_FIRST_ARG_ONLY = ("os.path.join",) def is_host_path(value): return isinstance(value, str) and value.startswith(HOST_PREFIXES) def is_pattern(value): """A bare prefix is a matcher, not a location: '/root/' names a namespace.""" return isinstance(value, str) and value in HOST_PREFIXES def mentions(value): return sum(value.count(p) for p in HOST_PREFIXES) if isinstance(value, str) else 0 def dotted(node): """sys.path.insert -> 'sys.path.insert', open -> 'open', else ''.""" try: return ast.unparse(node) except Exception: return "" def prose_ids(tree): """Ids of constants that are the whole of an expression statement.""" out = set() for node in ast.walk(tree): if isinstance(node, ast.Expr): val = node.value if isinstance(val, ast.Constant) and isinstance(val.value, str): out.add(id(val)) elif isinstance(val, ast.JoinedStr): for sub in ast.walk(val): if isinstance(sub, ast.Constant) and isinstance(sub.value, str): out.add(id(sub)) return out def parent_map(tree): out = {} for node in ast.walk(tree): for child in ast.iter_child_nodes(node): out[id(child)] = node return out def _ids(node): return {id(n) for n in ast.walk(node)} def subtype(node, pmap): """Classify a CODE path value by walking up to the call or binding it feeds.""" cur = node for _ in range(4): p = pmap.get(id(cur)) if p is None: return "CONST" if isinstance(p, ast.Call): name = dotted(p.func) if name in READ_FIRST_ARG_ONLY: first = p.args[0] if p.args else None return "READ" if first is not None and id(cur) in _ids(first) else "CONST" if name in BIND_FUNCS: return "BIND" if name in RUN_FUNCS: return "RUN" if name in READ_FUNCS: return "READ" if isinstance(p, ast.keyword) and p.arg == "cwd": return "RUN" if isinstance(p, ast.Assign): return "BIND" if any(dotted(t) == "sys.path" for t in p.targets) else "CONST" cur = p return "CONST" def scan_text(text, path=""): """Return (code_rows, prose_mentions, code_mentions, pattern_mentions).""" tree = ast.parse(text) prose, pmap = prose_ids(tree), parent_map(tree) code, prose_n, code_n, pattern_n = [], 0, 0, 0 for node in ast.walk(tree): if not (isinstance(node, ast.Constant) and isinstance(node.value, str)): continue hit = mentions(node.value) if not hit: continue if id(node) in prose: prose_n += hit elif is_pattern(node.value): pattern_n += hit elif is_host_path(node.value): code.append({"path": path, "line": node.lineno, "value": node.value, "subtype": subtype(node, pmap)}) else: code_n += hit return code, prose_n, code_n, pattern_n def inventory(root): out = subprocess.run(["git", "-C", root, "ls-files", "-z", "--", "wizard-work"], capture_output=True) if out.returncode != 0: raise RuntimeError(f"git ls-files rc {out.returncode}: " f"{out.stderr.decode(errors='replace')[:200]}") return sorted(p for p in out.stdout.decode().split("\0") if p.endswith(".py")) def lane_of(path): parts = path.split("/") return "/".join(parts[:2]) if len(parts) > 2 else parts[0] def census(root, lane_filter=""): files = inventory(root) if lane_filter: files = [f for f in files if f == lane_filter or f.startswith(lane_filter + "/")] rows, unreadable = [], [] for rel in files: try: with tokenize.open(os.path.join(root, rel)) as fh: text = fh.read() code, prose, code_n, pattern_n = scan_text(text, rel) except (OSError, UnicodeDecodeError, SyntaxError) as exc: unreadable.append({"path": rel, "error": f"{type(exc).__name__}: {exc}"}) continue rows.append({"path": rel, "lane": lane_of(rel), "code": code, "prose": prose, "code_mentions": code_n, "pattern": pattern_n}) lanes = {} for row in rows: agg = lanes.setdefault(row["lane"], {"files": 0, "code_files": 0, "code": 0, "prose": 0, "prose_only": 0, "pattern": 0}) agg["files"] += 1 agg["prose"] += row["prose"] agg["pattern"] += row["pattern"] agg["code"] += len(row["code"]) if row["code"]: agg["code_files"] += 1 elif row["prose"]: agg["prose_only"] += 1 return files, rows, lanes, unreadable def report(files, rows, lanes, unreadable, list_rows): print(f"q69 host-root census -- {len(files)} tracked wizard-work/*.py") print(f" {'lane':<26}{'files':>7}{'code-files':>12}{'code':>7}{'prose':>7}" f"{'prose-only':>12}{'pattern':>9}") for lane in sorted(lanes, key=lambda k: (-lanes[k]["code_files"], k)): a = lanes[lane] print(f" {lane:<26}{a['files']:>7}{a['code_files']:>12}{a['code']:>7}" f"{a['prose']:>7}{a['prose_only']:>12}{a['pattern']:>9}") tot_code = sum(len(r["code"]) for r in rows) tot_prose = sum(r["prose"] for r in rows) tot_pattern = sum(r["pattern"] for r in rows) code_files = sum(1 for r in rows if r["code"]) prose_only = sum(1 for r in rows if not r["code"] and r["prose"]) print(f" {'TOTAL':<26}{len(rows):>7}{code_files:>12}{tot_code:>7}{tot_prose:>7}" f"{prose_only:>12}{tot_pattern:>9}") if list_rows: for row in rows: for hit in row["code"]: print(f" {hit['subtype']:<6} {hit['path']}:{hit['line']} {hit['value']}") for bad in unreadable: print(f" UNGRADED {bad['path']} {bad['error']}") return tot_code, code_files, tot_prose, prose_only # ---------------------------------------------------------------- selftest CELLS = [ # name, source, want_code, want_prose, want_mentions, want_subtype, want_pattern ("docstring mid-sentence", '"""reads /root/troupe-standup/x.py once"""\n', 0, 1, 0, None, 0), ("comment only", '# /root/troupe-standup/x.py\nx = 1\n', 0, 0, 0, None, 0), ("sys.path insert", 'import sys\nsys.path.insert(0, "/root/.hermes/x")\n', 1, 0, 0, "BIND", 0), ("module constant", 'REPO = "/root/troupe-standup"\n', 1, 0, 0, "CONST", 0), ("constant then open", 'REPO = "/root/troupe-standup"\nopen(REPO + "/x")\n', 1, 0, 0, "CONST", 0), ("open literal", 'open("/root/troupe-standup/x")\n', 1, 0, 0, "READ", 0), ("argv element", 'import subprocess\nsubprocess.run(["bash", "/root/x/run.sh"])\n', 1, 0, 0, "RUN", 0), ("cwd keyword", 'import subprocess\nsubprocess.run(["ls"], cwd="/root/troupe-standup")\n', 1, 0, 0, "RUN", 0), ("sys.path assign", 'import sys\nsys.path = ["/root/x"] + sys.path\n', 1, 0, 0, "BIND", 0), ("code message mention", 'print("imported from /root/troupe-standup/x")\n', 0, 0, 1, None, 0), ("host tool path ignored", 'X = "/usr/bin/env"\nY = "/tmp/x"\n', 0, 0, 0, None, 0), # The census's own matcher table: six prefixes equal to a bare host prefix # are PATTERN, not six path values and not a docstring (Q69 self-count). ("bare prefix is a pattern", 'P = ("/root/", "/home/")\n', 0, 0, 0, None, 2), ] def selftest(): failed = 0 for name, text, want_code, want_prose, want_mention, want_sub, want_pat in CELLS: code, prose, mention, pattern = scan_text(text) subs = [c["subtype"] for c in code] ok = (len(code) == want_code and prose == want_prose and mention == want_mention and pattern == want_pat and (want_sub is None or subs == [want_sub])) failed += 0 if ok else 1 print(f" {'ok ' if ok else 'FAIL'} {name}: code={len(code)} prose={prose} " f"mentions={mention} pattern={pattern} subs={subs}") print(f" {len(CELLS)} cell(s), {failed} failed") return 0 if failed == 0 else 1 def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo", default=os.path.dirname(os.path.dirname( os.path.dirname(os.path.abspath(__file__))))) ap.add_argument("--lane", default="") ap.add_argument("--list", action="store_true") ap.add_argument("--json", default="") ap.add_argument("--selftest", action="store_true") args = ap.parse_args() if args.selftest: return selftest() try: files, rows, lanes, unreadable = census(args.repo, args.lane) except (RuntimeError, OSError) as exc: print(f"CANNOT GRADE: {exc}") return 2 if args.lane and not files: print(f"CANNOT GRADE: no tracked *.py under {args.lane}") return 1 code, code_files, prose, prose_only = report(files, rows, lanes, unreadable, args.list) if args.json: try: with open(args.json, "w", encoding="utf-8") as fh: json.dump({"files": len(files), "code_constants": code, "code_files": code_files, "prose_mentions": prose, "prose_only_files": prose_only, "pattern_constants": sum(r["pattern"] for r in rows), "code_mention_constants": sum(r["code_mentions"] for r in rows), "lanes": lanes, "unreadable": unreadable, "rows": [r for r in rows if r["code"]]}, fh, indent=2) print(f"wrote {args.json}") except OSError as exc: print(f"CANNOT GRADE: cannot write {args.json}: {exc}") return 2 if unreadable: print(f"CANNOT GRADE: {len(unreadable)} file(s) unreadable or unparsed") return 2 print(f"census: {code_files} code file(s), {prose_only} prose-only file(s), " f"{len(files)} graded -- rc 0") return 0 if __name__ == "__main__": raise SystemExit(main())