troupe-standup/wizard-work/quill/verify_placeholder_ids.py

#!/usr/bin/env python3
"""verify_placeholder_ids.py — verify the placeholder-ID standard and its own detection.

Scans:
  1. The standard document itself for bare invented IDs.
  2. Embedded unit tests that exercise the detection logic.

Exits 0 on clean pass; exits 1 on any failure.
"""
import re, os, sys

REPO_ROOT = "/root/troupe-standup"
ROOTS = [
    "/root/.hermes/wizard-outputs/quill/",
    "/root/troupe-standup/wizard-work/quill/",
]
STANDARD = "/root/.hermes/wizard-outputs/quill/placeholder-id-format-standard.md"

# Build existing id set from filesystem (repo + scanned roots)
existing_ids = set()
for scan_root in [REPO_ROOT] + ROOTS:
    root_name = os.path.basename(scan_root.rstrip("/"))
    if root_name and not root_name.startswith("."):
        existing_ids.add(root_name)
    for dirpath, dirnames, filenames in os.walk(scan_root):
        rel = os.path.relpath(dirpath, scan_root)
        if rel == ".":
            rel = ""
        for d in dirnames:
            if d.startswith(".") or d == "__pycache__":
                continue
            existing_ids.add(d)
            if rel:
                existing_ids.add(f"{rel}/{d}")
        for f in filenames:
            if f.startswith("."):
                continue
            stem = os.path.splitext(f)[0]
            existing_ids.add(stem)
            existing_ids.add(f)
            if rel:
                existing_ids.add(f"{rel}/{f}")
                existing_ids.add(f"{rel}/{stem}")

# Common English hyphenated words to skip
STOP = {
    "free-tier", "on-call", "one-line", "one-sentence", "in-flight", "low-risk",
    "non-zero", "non-empty", "user-facing", "user-visible", "end-to-end",
    "time-boxed", "long-task", "re-run", "re-scan", "back-to-back", "built-in",
    "high-value", "open-plan", "night-owls", "self-hosted", "real-world",
    "long-running", "large-scale", "real-time", "evidence-based", "evidence-backed",
    "deep-work", "single-day", "known-good", "last-known", "mean-time",
    "flash-class", "open-weight", "free-llm", "relay-only", "browser-relay",
    "same-page", "full-time", "part-time", "cross-functional", "cross-tool",
    "cross-platform", "hard-coded", "hard-wired", "real-estate", "all-in",
    "out-of-the-box", "out-of-band", "fine-grained", "real-life",
    "best-practice", "best-in-class", "first-class", "next-gen",
    "next-generation", "world-class", "cutting-edge", "bleeding-edge",
    "production-ready", "enterprise-grade", "battle-tested", "cloud-native",
    "data-driven", "insight-driven", "value-driven", "result-driven",
    "goal-oriented", "task-oriented", "outcome-oriented", "solution-oriented",
    "customer-centric", "product-centric", "design-centric", "tech-centric",
    "human-readable", "machine-readable", "human-friendly", "user-friendly",
    "cloud-based", "web-based", "api-based", "service-based",
    "microservice-based", "serverless-based", "container-based",
    "full-stack", "front-end", "back-end", "middleware",
    "end-user", "power-user", "super-user", "admin-user",
    "check-in", "check-out", "log-in", "log-out", "sign-up", "sign-in",
    "opt-in", "opt-out", "drop-in", "fall-back", "set-up",
    "follow-up", "catch-up", "keep-up", "give-away", "give-back",
    "break-down", "break-through", "break-up",
    "work-around", "work-through", "work-out", "work-up",
    "think-through", "think-out", "figure-out", "point-out",
    "carry-out", "carry-on", "carry-over", "carry-through",
    "come-up", "come-through", "come-back", "come-down",
    "run-down", "run-through", "run-up", "run-off",
    "turn-around", "turn-away", "turn-out", "turn-up",
    "look-up", "look-out", "look-over", "look-through",
    "make-up", "make-out", "make-over",
    "put-together", "put-up", "put-down", "put-off",
    "get-together", "get-up", "get-out", "get-off", "get-on", "get-in",
    "take-out", "take-up", "take-down", "take-off", "take-on", "take-in",
    "hand-off", "hand-over", "hand-out", "hand-up",
    "stand-up", "stand-out", "stand-by", "stand-off",
    "show-up", "show-off",
    "fall-back", "fall-out", "fall-through", "fall-off",
    "call-out", "call-up", "call-off", "call-in",
    "drop-off", "drop-out", "drop-by", "drop-in",
    "pass-through", "pass-out", "pass-up", "pass-off",
    "fail-safe", "fail-over", "fail-fast", "soft-landing",
    "hard-landing", "cold-start", "warm-up", "cool-down",
    "heat-up", "speed-up", "slow-down", "scale-up", "scale-down",
    "grow-up", "grow-out", "grow-in",
    "roll-out", "roll-in", "roll-up", "roll-down",
    "spin-up", "spin-down", "spin-off", "spin-out",
    "boot-up", "boot-down", "shut-down", "power-up", "power-down",
    "fire-up", "fire-off", "blow-up", "blow-out", "blow-over",
    "clean-up", "clean-out", "clean-off", "clean-down",
    "clear-out", "clear-up", "clear-off",
    "fill-in", "fill-out", "fill-up",
    "top-up", "top-down", "bottom-up",
    "up-to-date", "up-to-speed", "up-to-scratch",
    "out-of-date", "out-of-order", "out-of-control",
    "out-of-scope", "in-scope", "in-line", "in-place", "in-order",
    "in-turn", "in-effect", "in-force", "in-action",
    "on-board", "off-board", "off-site", "on-site",
    "on-prem", "off-prem", "off-grid", "on-grid",
    "at-scale", "at-risk", "at-large", "at-hand",
    "by-default", "by-design", "by-force", "by-law",
    "for-free", "for-sale", "for-rent", "for-keep",
    "from-scratch", "from-source", "from-home",
    "into-action", "into-effect", "into-place",
    "near-term", "near-future", "far-future", "far-reaching",
    "high-level", "low-level", "mid-term", "mid-range", "mid-point",
    "long-term", "short-term", "long-run", "short-run",
    "raw-data", "raw-feed", "raw-output",
    "small-scale", "large-scale", "small-size", "large-size",
    "full-size", "half-size", "double-size", "triple-size",
    "single-source", "single-point", "single-page", "single-line",
    "multi-source", "multi-point", "multi-page", "multi-line",
    "cross-reference", "cross-check", "cross-post", "cross-link",
    "peer-to-peer", "point-to-point",
    "many-to-many", "one-to-one", "one-to-many", "many-to-one",
    "all-or-nothing", "more-or-less", "less-is-more",
    "more-than", "less-than", "greater-than", "equal-to",
    "first-come", "first-served", "first-rate",
    "second-class", "third-class", "top-class", "top-notch",
    "world-wide", "country-wide", "city-wide",
    "company-wide", "team-wide", "department-wide",
    "industry-wide", "market-wide", "region-wide", "global-scale",
    "local-scale", "regional-scale", "national-scale",
    "front-line", "back-line", "side-line", "end-line",
    "hard-line", "soft-line", "bottom-line", "top-line",
    "red-line", "blue-line", "green-line", "yellow-line",
    "fine-line", "thin-line", "thick-line",
    "cut-line", "dotted-line", "dash-line",
    "head-line", "sub-head", "tag-line", "by-line",
    "dead-line", "life-line", "time-line",
    "story-line", "plot-line", "theme-line",
    "birth-date", "death-date", "due-date", "start-date", "end-date",
    "launch-date", "release-date", "update-date", "create-date",
    "modify-date", "change-date", "check-date", "test-date",
    "run-date", "build-date", "deploy-date", "ship-date",
    "close-date", "open-date", "book-date", "cancel-date",
    "plan-date", "review-date", "approve-date", "reject-date",
    "post-date", "pre-date", "re-date",
    "double-check", "triple-check",
    "fact-check", "sanity-check", "reality-check",
    "health-check", "sense-check", "smoke-check",
    "quick-check", "deep-check", "full-check", "partial-check",
    "self-check", "peer-check", "auto-check", "manual-check",
    "system-check", "unit-check", "integration-check", "e2e-check",
    "acceptance-check", "compliance-check", "security-check",
    "performance-check", "load-check", "stress-check",
    "regression-check",
    "pre-check", "post-check", "mid-check",
    "code-check", "style-check", "lint-check", "format-check",
    "type-check", "logic-check", "data-check", "input-check",
    "output-check", "config-check", "env-check", "path-check",
    "file-check", "dir-check", "folder-check", "link-check",
    "url-check", "reference-check", "cite-check",
    "source-check", "evidence-check", "proof-check",
    "sign-check", "crc-check", "hash-check", "checksum-check",
    "backup-check", "restore-check", "migration-check",
    "upgrade-check", "downgrade-check", "patch-check",
    "rollback-check", "failover-check", "recovery-check",
    "repair-check", "fix-check", "update-check", "refresh-check",
    "re-index", "re-load", "re-read", "re-write", "re-do", "re-make",
    "re-build", "re-deploy", "re-start", "re-boot",
    "re-connect", "re-link", "re-attach", "re-join",
    "re-enter", "re-import", "re-export", "re-sort", "re-filter",
    "re-parse", "re-encode", "re-decode", "re-paint", "re-draw",
    "re-render", "re-size", "re-scale", "re-shape", "re-form",
    "re-format", "re-struct", "re-arch", "re-design", "re-model",
    "re-train", "re-fit", "re-predict", "re-infer",
    "re-query", "re-fetch", "re-call", "re-invoke",
    "re-trigger", "re-fire", "re-kick", "re-push",
    "re-pull", "re-merge", "re-base", "re-commit", "re-tag",
    "re-branch", "re-checkout", "re-clone", "re-fork", "re-spawn",
    "re-launch", "re-run",
}

ID_RE = re.compile(r"\b[a-z0-9]+(?:-[a-z0-9]+)+\b")
BACKTICK_RE = re.compile(r"`([^`]+)`")
PATH_IN_TEXT_RE = re.compile(r"(?:^|[\s(])((?:\.{1,2}/|/root/)[^\s)`]+)")
REL_PATH_RE = re.compile(r"(?:^|[\s(])((?:wizard-work|wizard-outputs)/[^\s)`]+)")
REGEX_META_RE = re.compile(r'[\[\]\(\)\{\}\+\*\?\\\^\$\.\|]')

def token_in_ref_context(line, start, end):
    """True if the token is inside backticks, a path, or a list/table cell."""
    for m in BACKTICK_RE.finditer(line):
        if start >= m.start(1) and end <= m.end(1):
            return True
    for pm in PATH_IN_TEXT_RE.finditer(line):
        if start >= pm.start(1) and end <= pm.end(1):
            return True
    for pm in REL_PATH_RE.finditer(line):
        if start >= pm.start(1) and end <= pm.end(1):
            return True
    stripped = line.strip()
    if stripped.startswith(("-", "*", "|")):
        return True
    return False

def is_regex_like(token, line, start, end):
    """True if the token is inside backticks and looks like a regex/pattern."""
    for m in BACKTICK_RE.finditer(line):
        if start >= m.start(1) and end <= m.end(1):
            inner = m.group(1)
            # extract the token from inside the backticks
            idx = start - m.start(1)
            snippet = inner[max(0, idx-5):idx+len(token)+5]
            if REGEX_META_RE.search(snippet):
                return True
    return False

def scan_text(lines, source_label):
    violations = []
    in_code = False
    for lineno, line in enumerate(lines, 1):
        stripped = line.strip()
        if stripped.startswith("```"):
            in_code = not in_code
            continue
        if in_code:
            continue
        if "[PLACEHOLDER:" in line or "<PLACEHOLDER:" in line:
            continue
        for m in ID_RE.finditer(line):
            token = m.group(0)
            if token.lower() in STOP:
                continue
            start, end = m.span(0)
            if "<" in line[max(0, start-2):start] or ">" in line[end:end+2]:
                continue
            if not token_in_ref_context(line, start, end):
                continue
            if is_regex_like(token, line, start, end):
                continue
            if token not in existing_ids:
                violations.append((source_label, lineno, token, line.strip()))
    return violations

# 1. Scan the standard document itself
std_lines = open(STANDARD, "r", encoding="utf-8", errors="ignore").read().splitlines()
std_violations = scan_text(std_lines, STANDARD)

# 2. Embedded unit tests
test_cases = [
    ("clean backticked real ID", "See `queue-triage` for details.", 0),
    ("clean backticked placeholder", "See `<fleet-ops>` for details.", 0),
    ("clean backticked placeholder marker", "See `[PLACEHOLDER: fleet-ops]` for details.", 0),
    ("violation bare invented in backtick", "See `fleet-ops` for details.", 1),
    ("violation bare invented in list", "- fleet-ops for details.", 1),
    ("clean English hyphen", "This is a free-tier service.", 0),
    ("clean English hyphen in list", "- free-tier service.", 0),
]

test_violations = []
for desc, line, expected in test_cases:
    result = scan_text([line], f"test:{desc}")
    actual = len(result)
    if actual != expected:
        test_violations.append((desc, expected, actual, result))

all_violations = std_violations + test_violations

if all_violations:
    print(f"VIOLATIONS: {len(all_violations)}")
    for item in all_violations:
        if len(item) == 4:
            src, ln, token, ctx = item
            print(f"  {src}:{ln}: bare ID '{token}' not in repo")
            print(f"    context: {ctx[:120]}")
        else:
            desc, exp, act, res = item
            print(f"  TEST FAIL: {desc} (expected {exp}, got {act})")
            for r in res:
                print(f"    {r}")
    sys.exit(1)
else:
    print("PASS: zero bare-ID violations found; all embedded tests passed.")
    sys.exit(0)