#!/usr/bin/env python3
"""Independent verification for the workqueue-census artifact.

RE-APPROACH: this checker does NOT import the extractor. It re-derives the
expected rows from WORK-QUEUE.md with plain string operations (split on
whitespace, HEADING_RE) and compares them to the committed TSV and manifest.

TWO FAULTS THIS FILE CARRIED, AND THEIR REPAIR (Q112, 2026-10-29)
  1. THE READER. `read_tsv()` split each line on "\\t". The extractor writes
     the TSV with `csv.writer(delimiter="\\t")`, whose QUOTE_MINIMAL dialect
     wraps a cell containing a `"` in double quotes and doubles the inner
     quote (`"the ""open"" line"`). A naive split therefore handed the
     comparison a leading `"` and an un-doubled tail, and the committed
     artifact read 11 false faults at HEAD -- rows 43, 45, 52, 60, 64, 77,
     81, 84, 87, 88, 96, measured to be exactly the cells whose true value
     holds a quote. The reader now parses the dialect the writer writes;
     `read_tsv_naive()` keeps the old reader so the selftest can measure what
     it cost instead of asserting that it would have cost it.
  2. THE TAIL LINE. The revision leg printed `PASS source revision ...
     matches HEAD` unconditionally -- including on runs where it had just
     printed `SKIP revision leg`. A leg that did not run was reported with the
     same word as one that did. Revision is now graded as a FORMAT fact
     (12 hex digits or the literal `unknown`) and both values are printed on
     one INFO line; nothing here decays with the wall clock or with HEAD's
     movement.
  3. THE SAME CLASS, CAUGHT IN THIS REPAIR'S OWN FIRST DRAFT: that draft's row
     leg printed `PASS rows: ... 1 cell fault(s)` -- the count contradicting
     the word -- as measured on a seeded cell mutation before it shipped. Every
     leg line here now takes its word from the counter it prints, and lists
     the denominators beside it.

THE LEG Q112 ASKED FOR: `record_only_ids`. Some ids open a record in bold-lead
form (`**Qnn -- ...**`) and never appear as a `## Qnn` heading, so a
heading-only walk cannot see them: at this head the queue opens 122 ids, the
heading walk sees 86, and 36 are record-form only. The extractor and this
checker each compute the set from their own scan; the manifest must carry it
verbatim, and the count of ids opened in any form must reconcile with it.

Verdict: the script's exit code. 0 = correct, 1 = a check failed, 2 = could not
grade (missing artifact / bad invocation).

Usage:
  python3 CHECK                        # grade this directory against its tree
  python3 CHECK --root DIR             # grade a tree other than the one this sits in
  python3 CHECK --artifact DIR         # grade a census directory other than this one
  python3 CHECK --selftest             # build fixture trees and mutate one thing each
"""

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
DEFAULT_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
ARTIFACT_REL = os.path.join("wizard-work", "ledger", "2026-10-28-workqueue-census")

HEADING_RE = re.compile(r"^##\s+Q(\d+)(.*)$")
STATUS_RE = re.compile(r"^\*\*Status:\s*(.*)$")
DONE_RE = re.compile(r"^\*\*Done means:\s*(.*)$")

# Record-open forms that are NOT headings (the rule stated in WORK-QUEUE.md at
# Q63: an id is opened by a record, not mentioned by prose). A bold run opens
# an id only when the id is the first token after the bullet/bold markers and
# the next token is a separator or the word open/opened -- `**Q7, measured:**`
# is a mention and must not count.
BOLD_LEAD = re.compile(r"^[\s>|]*(?:[-*+]+\s+)?\*\*(Q\d+)(.*)$")
OPEN_AFTER = re.compile(r"^\s*(?:—|–|-|:|\(|\bopened\b|\bopen\b|\bOPEN\b|\bOPENED\b)")
STATUS_LEAD = re.compile(r"^\s*\*{0,2}Status:\s*(Q\d+)\b")

TSV_HEADER = ["q", "title", "status", "done_implies"]
TSV_DELIMITER = "\t"
TS_FMT = "%Y-%m-%dT%H:%M:%SZ"
REV_RE = re.compile(r"^(?:[0-9a-f]{12}|unknown)$")
SCHEMA = "workqueue-census-v3"
SEP, DASH = "—", "–"


def collapse(s):
    return re.sub(r"\s+", " ", s).strip()


def _truncate_sentence(s):
    """Collapse whitespace and cut at the first sentence end, one trailing '.'."""
    s = collapse(s)
    m = re.search(r"\.\s", s)
    if m:
        s = s[: m.start()] + "."
    if s.endswith("."):
        s = s[: -1]
    return s + "."


def record_only_ids(lines, heading_ids):
    """Ids opened in bold-lead or status-lead form that have no heading."""
    seen = {}
    for line in lines:
        m = BOLD_LEAD.match(line)
        if m and m.group(1) not in heading_ids:
            if OPEN_AFTER.match(m.group(2)) and m.group(1) not in seen:
                seen[m.group(1)] = line
            continue
        m = STATUS_LEAD.match(line)
        if m and m.group(1) not in heading_ids and m.group(1) not in seen:
            seen[m.group(1)] = line
    return sorted(seen, key=lambda q: int(q[1:]))


def parse_source(path):
    """Independent re-derivation of the census from WORK-QUEUE.md."""
    with open(path, "r", encoding="utf-8") as fh:
        lines = fh.read().splitlines()
    rows = []
    heading_ids = set()
    current = None
    for line in lines:
        m = HEADING_RE.match(line)
        if m:
            heading_ids.add("Q" + m.group(1))
            if current is not None:
                rows.append(current)
            current = {
                "q": "Q" + m.group(1),
                "title": m.group(2).strip(),
                "status": None,
                "done_implies": None,
            }
        elif current is not None:
            sm = STATUS_RE.match(line)
            if current["status"] is None and sm:
                current["status"] = _truncate_sentence(sm.group(1))
            dm = DONE_RE.match(line)
            if current["done_implies"] is None and dm:
                current["done_implies"] = collapse(dm.group(1))
    if current is not None:
        rows.append(current)
    for r in rows:
        if r["status"] is None:
            r["status"] = ""
        if r["done_implies"] is None:
            r["done_implies"] = ""
    qids = [r["q"] for r in rows]
    freq = {}
    for q in qids:
        freq[q] = freq.get(q, 0) + 1
    reopened = sorted(q for q, n in freq.items() if n > 1)
    only = record_only_ids(lines, heading_ids)
    opened_all = sorted(heading_ids | set(only), key=lambda q: int(q[1:]))
    return rows, len(rows), len(set(qids)), reopened, only, opened_all


def read_tsv(path):
    """Read the TSV in the dialect the extractor writes (csv, tab-delimited).

    QUOTE_MINIMAL is on, so a cell holding a `"` is quoted and its inner quotes
    doubled. Splitting on "\\t" (the pre-Q112 reader) cannot undo that.
    """
    import csv

    with open(path, "r", encoding="utf-8", newline="") as fh:
        data = list(csv.reader(fh, delimiter=TSV_DELIMITER))
    if not data:
        return [], []
    return data[0], data[1:]


def read_tsv_naive(path):
    """The pre-Q112 reader, kept so the selftest can measure what it cost."""
    with open(path, "r", encoding="utf-8") as fh:
        data = [line.split(TSV_DELIMITER) for line in fh.read().splitlines()]
    return data[0], data[1:]


def is_rev(value):
    return isinstance(value, str) and bool(REV_RE.match(value))


def _brief(value, width=120):
    text = repr(value)
    return text if len(text) <= width else text[:width] + "..."


def _parses_stamp(value):
    if not isinstance(value, str):
        return False
    import datetime

    try:
        datetime.datetime.strptime(value, TS_FMT)
    except ValueError:
        return False
    return True


def _git_head(root):
    try:
        out = subprocess.run(["git", "-C", root, "rev-parse", "--short=12", "HEAD"],
                             capture_output=True, text=True)
        if out.returncode == 0:
            return out.stdout.strip()
    except Exception:  # noqa: BLE001
        pass
    return "unknown"


def grade(root, artifact, reader=None, say=print):
    """Grade one census directory against the tree it claims to extract."""
    reader = reader or read_tsv
    problems = []
    src = os.path.join(root, "WORK-QUEUE.md")
    tsv_path = os.path.join(artifact, "workqueue-census.tsv")
    man_path = os.path.join(artifact, "extract_manifest.json")

    try:
        (src_rows, n_headings, n_distinct, reopened,
         record_only, opened_all) = parse_source(src)
    except Exception as exc:  # noqa: BLE001
        say("CANNOT GRADE source %s: %s" % (src, exc))
        return 2

    if not os.path.exists(tsv_path):
        say("FAIL missing TSV %s" % tsv_path)
        return 1
    try:
        header, tsv_rows = reader(tsv_path)
    except Exception as exc:  # noqa: BLE001
        say("CANNOT GRADE TSV %s: %s" % (tsv_path, exc))
        return 2

    # --- leg: the rows reproduce the source walk, cell for cell -------------
    cell_faults = 0
    if header != TSV_HEADER:
        problems.append("header %r != %r" % (header, TSV_HEADER))
        cell_faults += 1
    if len(tsv_rows) != n_headings:
        problems.append("TSV rows %d != headings %d" % (len(tsv_rows), n_headings))
        cell_faults += 1
    for i, (expect, actual) in enumerate(zip(src_rows, tsv_rows)):
        if len(actual) != 4:
            problems.append("row %d has %d fields" % (i, len(actual)))
            cell_faults += 1
            continue
        for col, key in enumerate(TSV_HEADER):
            if actual[col] != expect[key]:
                problems.append("row %d (%s) %s mismatch" % (i, expect["q"], key))
                cell_faults += 1
    say("%s rows: %d heading(s), %d TSV row(s), %d cell fault(s) on the %d "
        "re-derived row(s)"
        % ("PASS" if cell_faults == 0 else "FAIL", n_headings, len(tsv_rows),
           cell_faults, len(src_rows)))

    # --- leg: the manifest reconciles with the recomputation above ----------
    if not os.path.exists(man_path):
        say("FAIL missing manifest %s" % man_path)
        return 1
    try:
        with open(man_path, "r", encoding="utf-8") as fh:
            manifest = json.load(fh)
    except Exception as exc:  # noqa: BLE001
        say("CANNOT GRADE manifest %s: %s" % (man_path, exc))
        return 2

    legs = [
        ("schema", manifest.get("schema"), SCHEMA),
        ("rows", manifest.get("rows"), n_headings),
        ("distinct_q_labels", manifest.get("distinct_q_labels"), n_distinct),
        ("reopened_q_labels", manifest.get("reopened_q_labels"), reopened),
        ("record_only_ids", manifest.get("record_only_ids"), record_only),
        ("ids_opened_total", manifest.get("ids_opened_total"), len(opened_all)),
    ]
    bad_legs = 0
    for name, got, want in legs:
        if got != want:
            problems.append("manifest %s %r != recomputed %r"
                            % (name, _brief(got), _brief(want)))
            bad_legs += 1
    say("%s manifest: %d of %d leg(s) reconciled, %d fault(s) (%d record-form-only "
        "id(s), %d id(s) opened in any form)"
        % ("PASS" if bad_legs == 0 else "FAIL", len(legs) - bad_legs, len(legs),
           bad_legs, len(record_only), len(opened_all)))

    # --- leg: provenance that cannot decay ---------------------------------
    prov_faults = 0
    if not _parses_stamp(manifest.get("extracted_at_utc")):
        problems.append("extracted_at_utc %r is not %s"
                        % (manifest.get("extracted_at_utc"), TS_FMT))
        prov_faults += 1
    if not is_rev(manifest.get("revision")):
        problems.append("revision %r is not 12 hex digits or 'unknown'"
                        % (manifest.get("revision"),))
        prov_faults += 1
    say("%s provenance: %d fault(s) (stamp shape %s, revision shape %s)"
        % ("PASS" if prov_faults == 0 else "FAIL", prov_faults,
           "ok" if _parses_stamp(manifest.get("extracted_at_utc")) else "bad",
           "ok" if is_rev(manifest.get("revision")) else "bad"))
    say("INFO manifest revision=%s head=%s -- revision equality is not graded: the "
        "queue is appended to by other lanes between generation and reading"
        % (manifest.get("revision"), _git_head(root)))

    # --- leg: TSV sanity (no empty q/title) --------------------------------
    for i, row in enumerate(tsv_rows, start=1):
        if len(row) < 4:
            problems.append("row %d too short" % i)
        elif row[0].strip() == "":
            problems.append("row %d empty q" % i)
        elif row[1].strip() == "":
            problems.append("row %d empty title" % i)

    if problems:
        say("FAIL (%d problem(s))" % len(problems))
        for p in problems:
            say("  - %s" % p)
        return 1
    say("PASS census: %d rows, %d distinct labels, %d reopened, %d record-form only"
        % (n_headings, n_distinct, len(reopened), len(record_only)))
    return 0


# --------------------------------------------------------------------------
# selftest: build fixture trees, mutate one thing per cell, grade each.
# --------------------------------------------------------------------------
FIXTURE_QUEUE = """# fixture queue

## Q1 — first item

**Status:** done at `abc1234`.

**Done means:** the first cell reads back.

## Q2 — second item — the "pin" class

**Status:** open and unclaimed.

**Done means:** the second cell reads back.

## Q3 — third item

**Status:** open.

**Done means:** the third cell reads back.

**Q4 — OPEN (fixture lane; bold-lead record, no heading).**

**Q5 opened — another record-form id.**
"""


def _write(path, text):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8", newline="") as fh:
        fh.write(text)


def _read(path):
    with open(path, "r", encoding="utf-8") as fh:
        return fh.read()


def _build_tree(tmp, name, queue_text):
    """A tree the shipped resolution recognises, with the extractor beside it."""
    root = os.path.join(tmp, name)
    art = os.path.join(root, ARTIFACT_REL)
    os.makedirs(art)
    _write(os.path.join(root, "WORK-QUEUE.md"), queue_text)
    shutil.copy(os.path.join(HERE, "extract_workqueue_census.py"), art)
    env = dict(os.environ, REPO_ROOT=root)
    out = subprocess.run([sys.executable, "extract_workqueue_census.py"], cwd=art,
                         env=env, capture_output=True, text=True)
    if out.returncode != 0:
        raise RuntimeError("extractor failed in %s: %s" % (root, out.stderr.strip()))
    return root, art


def selftest(say=print):
    cells = []
    tmp = tempfile.mkdtemp(prefix="census-selftest-")
    quiet = lambda *a: None  # noqa: E731
    try:
        # cell 0 -- control: the fixture, generated by the extractor, is green.
        root, art = _build_tree(tmp, "control", FIXTURE_QUEUE)
        cells.append(("control-green", 0, grade(root, art, say=quiet)))

        # cell 1 -- a quoted cell reads back intact; the same tree under the
        # pre-Q112 reader is the falsification, not an assertion about it.
        root1, art1 = _build_tree(tmp, "quote", FIXTURE_QUEUE)
        cells.append(("quote-cell-green-with-csv-reader", 0,
                      grade(root1, art1, say=quiet)))
        cells.append(("quote-cell-red-with-naive-reader", 1,
                      grade(root1, art1, reader=read_tsv_naive, say=quiet)))

        # cell 2 -- a heading deleted from the source, artifact untouched.
        root2, art2 = _build_tree(tmp, "no-heading", FIXTURE_QUEUE)
        q2 = _read(os.path.join(root2, "WORK-QUEUE.md"))
        q2 = q2.replace('## Q2 \u2014 second item \u2014 the "pin" class\n', "")
        _write(os.path.join(root2, "WORK-QUEUE.md"), q2)
        cells.append(("deleted-heading-red", 1, grade(root2, art2, say=quiet)))

        # cell 3 -- one title cell corrupted in the TSV.
        root3, art3 = _build_tree(tmp, "bad-cell", FIXTURE_QUEUE)
        tsv3 = os.path.join(art3, "workqueue-census.tsv")
        _write(tsv3, _read(tsv3).replace("\u2014 first item", "\u2014 FIRST item"))
        cells.append(("corrupted-cell-red", 1, grade(root3, art3, say=quiet)))

        # cell 4 -- a new record-form id in the source, manifest not regenerated.
        root4, art4 = _build_tree(tmp, "new-record-id", FIXTURE_QUEUE)
        with open(os.path.join(root4, "WORK-QUEUE.md"), "a", encoding="utf-8") as fh:
            fh.write("\n**Q6 \u2014 OPEN (fixture lane; a second record-form id).**\n")
        cells.append(("new-record-form-id-red", 1, grade(root4, art4, say=quiet)))

        # cell 5 -- the manifest's record_only_ids loses one entry.
        root5, art5 = _build_tree(tmp, "short-record-ids", FIXTURE_QUEUE)
        man5 = os.path.join(art5, "extract_manifest.json")
        blob = json.loads(_read(man5))
        blob["record_only_ids"] = blob["record_only_ids"][:-1]
        _write(man5, json.dumps(blob, indent=2) + "\n")
        cells.append(("manifest-record-ids-short-red", 1,
                      grade(root5, art5, say=quiet)))

        # cell 6 -- an unparseable provenance stamp.
        root6, art6 = _build_tree(tmp, "bad-stamp", FIXTURE_QUEUE)
        man6 = os.path.join(art6, "extract_manifest.json")
        blob = json.loads(_read(man6))
        blob["extracted_at_utc"] = "yesterday"
        _write(man6, json.dumps(blob, indent=2) + "\n")
        cells.append(("manifest-stamp-shape-red", 1, grade(root6, art6, say=quiet)))

        # cell 7 -- a revision that is not a revision (the leg that used to
        # print PASS without measuring anything).
        root7, art7 = _build_tree(tmp, "bad-rev", FIXTURE_QUEUE)
        man7 = os.path.join(art7, "extract_manifest.json")
        blob = json.loads(_read(man7))
        blob["revision"] = "HEAD~1"
        _write(man7, json.dumps(blob, indent=2) + "\n")
        cells.append(("manifest-revision-shape-red", 1, grade(root7, art7, say=quiet)))
    finally:
        shutil.rmtree(tmp, ignore_errors=True)

    failed = 0
    for name, want, got in cells:
        ok = want == got
        failed += 0 if ok else 1
        say("%s %s (want rc %d, got rc %d)" % ("PASS" if ok else "FAIL", name, want, got))
    say("SELFTEST %s (%d cells, %d failed)"
        % ("OK" if not failed else "FAILED", len(cells), failed))
    return 0 if failed == 0 else 1


def main(argv=None):
    ap = argparse.ArgumentParser(description="grade a workqueue census against its tree")
    ap.add_argument("--root", default=DEFAULT_ROOT, help="tree holding WORK-QUEUE.md")
    ap.add_argument("--artifact", default=HERE, help="census directory to grade")
    ap.add_argument("--selftest", action="store_true", help="mutate fixture trees")
    args = ap.parse_args(argv)
    if args.selftest:
        return selftest()
    return grade(os.path.abspath(args.root), os.path.abspath(args.artifact))


if __name__ == "__main__":
    sys.exit(main())