#!/usr/bin/env python3
"""queue_claim_check.py — the claim check for `WORK-QUEUE.md`, shipped with its tests.

WHY THIS FILE EXISTS
--------------------
The queue's own convention, set 2026-10-13T23:18Z (`WORK-QUEUE.md:8020`), says:

    "every new opening record carries a forward pointer to its closure line or
     revision, and a claim check reads this ledger before taking any item whose
     opening line reads open or unclaimed."

Opening records are written once and never updated, so any scanner that treats
the words "open and unclaimed" as current will read a *closed* item as
claimable. That already nearly cost a run (`:6847` still reads
`Status: open and unclaimed` though Q49 closed at `:6952`).

Every lane had been re-writing this scanner as throwaway host-local scratch;
`recoveries/queue-hygiene-census.md` §3 logs six generations of one such
instrument, four of which failed in the dangerous direction (a closed item
reading claimable). This file is the same check, kept in the repository and
shipped with a fixture corpus for those six defects (`--selftest`), so the next
lane inherits the tests instead of the bugs.

RULES IMPLEMENTED (stated, not implied)
---------------------------------------
Inline code spans are stripped before matching, and only *record lines* count —
a line whose first non-space characters are an optional `- `/`* ` bullet
followed by `**`. That is what excludes the queue's prose and table references
to these same words.

OPENING RECORD — a record line carrying either
  (a) `Status:` followed by `open`/`OPEN`, or
  (b) an item id `Q<n>` followed within 60 characters by an `—`/`–`/`--`
      separator and then `OPEN`/`open`/`Opened`/`accepted`.
Its item id is the leading `**Q<n>` of the line if present; otherwise the id is
inherited from the nearest preceding `## Q<n>` header.

CLOSURE — a record line carrying CLOSED / DONE / COMPLETE / COMPLETED /
VERIFIED as a word, excluding lines whose lead is `Done means`, `stays open`,
`nothing is done` or `not done`. It is anchored to an item by (in order): its
own leading `**Q<n>`; else any `Q<n>` token on the line; else the nearest
preceding `## Q<n>` header. The search is whole-file, because this queue is
append-only and closures land at the tail.

A closure line that anchors an opening record's item => that item is CLOSED.
An opening record with no anchored closure => OPEN. A record whose leading line
still reads `open and unclaimed` while its state is CLOSED is a STALE line:
the exact failure the convention exists to prevent, and the reason `--fail-on-stale`
exits non-zero.

KNOWN BLIND SPOTS (measured, not hand-waved)
--------------------------------------------
- Item ids are labels, not keys: this file's own record carries a collision
  (career-lane apply queue `:7292` also reads `Q53`; the queue's own reply 62
  notes it), which the tool reports as `COLLISION` rather than resolving.
- The state is only as good as the closure line's anchoring. Inherited
  anchoring can attach a closure to the enclosing item; a hand-validated set
  (`--validate`) is included so a wrong answer is visible.
- Nothing here judges whether a closure is *earned*, only that it exists.

USAGE
-----
    python3 tools/queue_claim_check.py                      # census of ./WORK-QUEUE.md
    python3 tools/queue_claim_check.py --queue PATH --json OUT.json
    python3 tools/queue_claim_check.py --validate           # + the hand-read answers
    python3 tools/queue_claim_check.py --git                # resolve closure revisions
    python3 tools/queue_claim_check.py --selftest           # fixture corpus, exit 0/1

Exit codes: 0 clean · 1 self-test failure · 2 stale opening line (--fail-on-stale)
            3 queue unreadable.  No file is written unless --json names one.
"""

import argparse
import json
import os
import re
import subprocess
import sys

# --------------------------------------------------------------------------
# matching primitives
# --------------------------------------------------------------------------

CODE_SPAN = re.compile(r'`[^`]*`')
RECORD_PREFIX = re.compile(r'^[ \t]*(?:[-*]\s+)?\*\*')
ITEM_LEAD = re.compile(r'^[ \t]*(?:[-*]\s+)?\*\*Q(\d+)\b')
HEADER_ITEM = re.compile(r'^#{1,3}\s+Q(\d+)\b')
Q_TOKEN = re.compile(r'\bQ(\d+)\b')

OPEN_STATUS = re.compile(r'Status:\s*\**\s*(?:open|OPEN)\b')
OPEN_Q = re.compile(
    r'Q\d+\b[^*\n]{0,60}?(?:\u2014|\u2013|--)\s*\**\s*(?:OPEN|open|Opened|accepted)\b')

CLOSURE_WORD = re.compile(r'\b(CLOSED|DONE|COMPLETE|COMPLETED|VERIFIED)\b')
CLOSURE_EXCLUDE = re.compile(r'^\s*(?:[-*]\s+)?\*\*\s*(?:Done means|stays open|nothing is done|not done)\b',
                             re.I)
CLOSURE_EXCLUDE_ANY = re.compile(r'stays open|nothing is done', re.I)

CONTAINER_HEAD = re.compile(r'^\s*(?:[-*]\s+)?\*\*Q(\d+)\b')
CONTAINER_TITLE_MAX = 100
CONTAINER_SEP = re.compile(r'(?:\u2014|\u2013|--|:)')

# A closure must be a *statement*: a record line whose lead is an item id or
# `Status:`, with the closure word inside the statement window. That is what
# refuses the queue's prose lines that mention a closure in passing
# (`:6631` "- **No item state changed this tick:** ... no DONE line", `:8430`
# "- **No item closed, no new item opened.** ... remains DONE at 32ad55b").
CLOSURE_LEAD = re.compile(r'^\s*(?:[-*]\s+)?\*\*(?:Q\d+\b|Status:)')
CLOSURE_WINDOW = 120

STALE_LITERAL = re.compile(r'Status:\s*\**\s*open and unclaimed', re.I)
SHA7 = re.compile(r'`([0-9a-f]{7,63})[0-9a-f\u2026]*`')

# Hand-read ground truth: eleven answers read off the queue by hand before any
# instrument runs (queue-hygiene-census.md §2). Two independent lanes now.
HAND_READ = {
    2264: 'CLOSED', 2684: 'CLOSED', 2898: 'CLOSED', 6348: 'CLOSED',
    6419: 'CLOSED', 6542: 'CLOSED', 6847: 'CLOSED', 6990: 'CLOSED',
    7106: 'CLOSED', 7167: 'CLOSED', 8530: 'OPEN',
}


def strip_code(line):
    return CODE_SPAN.sub('', line)


def is_record_line(line):
    return bool(RECORD_PREFIX.match(line))


def is_opening_record(line):
    s = strip_code(line)
    if not is_record_line(s):
        return False
    return bool(OPEN_STATUS.search(s) or OPEN_Q.search(s))


def is_closure_line(line):
    s = strip_code(line)
    if not CLOSURE_LEAD.match(s):
        return False
    if CLOSURE_EXCLUDE.match(s):
        return False
    return bool(CLOSURE_WORD.search(s[:CLOSURE_WINDOW]))


def lead_item(line):
    m = ITEM_LEAD.match(line)
    return int(m.group(1)) if m else None


def line_ids(line):
    """Every id a closure line can anchor to: its lead id, else its Q-tokens."""
    lead = lead_item(line)
    if lead is not None:
        return {lead}
    return {int(x) for x in Q_TOKEN.findall(strip_code(line))}


# --------------------------------------------------------------------------
# the census
# --------------------------------------------------------------------------

def item_title_container(line):
    """The id of a `**Q<n> — title**` container line, or None.

    A container is a *title statement*: bold, leading item id, a separator
    inside the bold span, and the bold span closing within CONTAINER_TITLE_MAX
    characters. Measured reasons, both from the live queue: `:5597`
    (`- **Q41 — the non-magazine excerpt class *and* placement** *(archive
    verification; ...`) is Q41's container and must count, while `:4457`
    (`**Q32 record precision — two of the thirteen new assertions cannot fail,
    ...`) is a prose record that must not — counting it stole Q33's own closure
    lines (`:4505`) for Q32.
    """
    m = CONTAINER_HEAD.match(line)
    if not m:
        return None
    rest = line[m.end():]
    close = rest.find('**')
    if close == -1 or close > CONTAINER_TITLE_MAX:
        return None
    if not CONTAINER_SEP.search(rest[:close]):
        return None
    return int(m.group(1))


def container_index(lines, loose=False):
    """[(line, id_or_None)] for every *container* line, in file order.

    A container is a header of any level, or a bold record line that leads with
    an item id. Two strictnesses, because the two questions differ:

    * `loose=True` — used to name an opening record that carries no id of its
      own. Any bold line leading with `Q<n>` counts; the item whose block the
      record sits in is the nearest such marker above it.
    * `loose=False` — used to anchor a closure. Here the marker must be a
      *title statement* (see `item_title_container`), because a long bold prose
      record that merely starts with an id steals the anchor: `:4457`
      (`**Q32 record precision — two of the thirteen new assertions cannot
      fail, ...`) otherwise took `:4505`, Q33's own closure, for Q32.
    """
    out = []
    for n, l in enumerate(lines, 1):
        if l.startswith('#'):
            m = HEADER_ITEM.match(l)
            out.append((n, int(m.group(1)) if m else None))
        elif loose and CONTAINER_HEAD.match(l):
            m = ITEM_LEAD.match(l)
            out.append((n, int(m.group(1)) if m else None))
        else:
            cid = item_title_container(l)
            if cid is not None:
                out.append((n, cid))
    return out


def inherit_id(n, hidx):
    """Id of the nearest preceding header, or None."""
    best = None
    for ln, qid in hidx:
        if ln < n:
            best = qid
        else:
            break
    return best


def census(text):
    lines = text.splitlines()
    hidx = container_index(lines, loose=False)
    lidx = container_index(lines, loose=True)

    closures = []          # (line, idset, text)
    unanchored = []        # closure statements that name no item
    for n, l in enumerate(lines, 1):
        if not is_closure_line(l):
            continue
        ids = line_ids(l)
        if not ids:
            inh = inherit_id(n, hidx)
            if inh is None:
                unanchored.append(n)
                continue
            ids = {inh}
        closures.append((n, ids, l))

    records = []
    for n, l in enumerate(lines, 1):
        if not is_opening_record(l):
            continue
        lead = lead_item(l)
        inherited = False
        if lead is None:
            lead = inherit_id(n, lidx)
            inherited = True
        anchored = [(cn, cl) for (cn, ids, cl) in closures if lead is not None and lead in ids]
        # State is decided by a closure recorded at or after the opening record.
        # Earlier closures are reported as evidence but never set the state: the
        # queue's record order is not the work's order (Q25's DONE line sits
        # 96 lines *above* its own opening record).
        closing = [c for c in anchored if c[0] >= n]
        state = 'CLOSED' if closing else 'OPEN'
        stale = bool(STALE_LITERAL.search(strip_code(l))) and state == 'CLOSED'
        records.append({
            'line': n,
            'item': None if lead is None else 'Q%d' % lead,
            'inherited_id': inherited,
            'state': state,
            'closure_line': closing[0][0] if closing else None,
            'closure_lines_after': [c[0] for c in closing],
            'closure_lines_before': [c[0] for c in anchored if c[0] < n],
            'closure_text': closing[0][1].strip() if closing else None,
            'stale': stale,
            'text': l.strip(),
        })

    # collisions: two opening records resolving to one id
    seen = {}
    for r in records:
        if r['item'] is None:
            continue
        seen.setdefault(r['item'], []).append(r['line'])
    for r in records:
        r['collision'] = bool(r['item'] and len(seen.get(r['item'], [])) > 1)

    raw_literal = len(re.findall(r'Status:\s*\**\s*open and unclaimed', text, re.I))
    return {
        'lines': len(lines),
        'bytes': len(text.encode()),
        'records': records,
        'closure_candidates': len(closures),
        'closure_unanchored': unanchored,
        'literal_anywhere': raw_literal,
    }


def annotate_revisions(rec, git_dir):
    """First 7-hex backticked token on the closure line, resolved with git."""
    out = []
    txt = rec.get('closure_text') or ''
    for tok in SHA7.findall(txt):
        try:
            p = subprocess.run(['git', 'cat-file', '-t', tok], cwd=git_dir,
                               capture_output=True, text=True, timeout=20)
            kind = p.stdout.strip() or p.stderr.strip()
            ok = p.returncode == 0
        except Exception as exc:                       # pragma: no cover
            kind, ok = str(exc), False
        out.append({'token': tok, 'resolves': ok, 'kind': kind})
    return out


# --------------------------------------------------------------------------
# fixtures — the six defects logged in recoveries/queue-hygiene-census.md §3
# --------------------------------------------------------------------------

FIXTURES = [
    ('v1 header-walk only: an opening record nested inside another section',
     '## Q10 — outer section\n\nDone means: nothing here.\n\n'
     '- **Q99 — OPEN (nested, not a `##` header).** body\n',
     ['Q99:OPEN']),

    ('v2 loose key: a Q7 continuation note is not an opening record',
     '## Q7 — section\n\n- **Q7, measured:** the unclaimed executable set is open beside it.\n',
     []),

    ('v3a a column-0 `---` rule does not cut the region',
     '## Q11 — section\n\n**Status: open and unclaimed.**\n\n---\n\n'
     '**Status: DONE — closed below the rule.**\n',
     ['Q11:CLOSED']),

    ('v3b `Done means:` is not a closure word',
     '## Q12 — section\n\n**Status: open and unclaimed.**\n\n**Done means:** one row per space.\n',
     ['Q12:OPEN']),

    ("v4 closures are anchored: another item's DONE does not close this one",
     '## Q13 — section\n\n**Status: open and unclaimed.**\n\n## Q14 — other\n\n'
     '**Q14 — OPEN (lane).**\n\n**Q14 — DONE at 4010b96.**\n',
     ['Q13:OPEN', 'Q14:CLOSED']),

    ('v5 whole-file search: a closure landing after a later opening record still counts',
     '## Q15 — section\n\n**Q15 — OPEN (lane).**\n\n## Q16 — other\n\n**Q16 — OPEN (lane).**\n\n'
     '**Q15 — DONE at 4010b96.**\n',
     ['Q15:CLOSED', 'Q16:OPEN']),

    ('grep trap: the same words in a code span or a table row are not an opening record',
     '## Q17 — section\n\n**Status: open and unclaimed. Opened by the pipe.**\n\n'
     '**Status: DONE — closed.**\n\n'
     '- (1) Three lines carry the literal `Status: open and unclaimed` (`:1`, `:2`).\n'
     '\n| `Status: open and unclaimed` | `:2264` | Q21 | DONE |\n',
     ['Q17:CLOSED']),

    ('unnumbered opening record inherits the enclosing `## Q<n>` header',
     '## Q20 — section\n\n**Status: OPEN — held for Alexander.**\n',
     ['Q20:OPEN']),

    ('a closed item whose opening line never got a pointer is flagged STALE',
     '## Q21 — section\n\n**Status: open and unclaimed. Opened by the pipe.**\n\n'
     '**Status: DONE — `recoveries/x.md`, 2026-10-10.**\n',
     ['Q21:CLOSED']),

    ('a bold prose record naming another item does not steal the container',
     '## Q30 — section\n\n**Status: open and unclaimed.**\n\n'
     '- **Q31 record precision — two of the thirteen new assertions cannot fail, and two totals are not\n'
     '  enumerable as stated**, so the record is corrected in place rather than restated.\n\n'
     '**Status: DONE — closed by the pipe.**\n',
     ['Q30:CLOSED']),
]

STALE_EXPECT = {
    'v3a a column-0 `---` rule does not cut the region': 1,
    'grep trap: the same words in a code span or a table row are not an opening record': 1,
    'a closed item whose opening line never got a pointer is flagged STALE': 1,
    'a bold prose record naming another item does not steal the container': 1,
}


def fixture_signature(text):
    res = census(text)
    sig = ['%s:%s' % (r['item'] or 'NONE', r['state']) for r in res['records']]
    return sig, len([r for r in res['records'] if r['stale']])


def selftest(verbose=True):
    failures = []
    for name, text, expect in FIXTURES:
        got, stale = fixture_signature(text)
        ok = got == expect
        want_stale = STALE_EXPECT.get(name, 0)
        if stale != want_stale:
            ok = False
        if verbose:
            print('%s %s' % ('PASS' if ok else 'FAIL', name))
            if not ok:
                print('     expected records %s (stale %d)\n     got      records %s (stale %d)'
                      % (expect, want_stale, got, stale))
        if not ok:
            failures.append(name)
    if verbose:
        print('\nself-test: %d fixtures, %d passed, %d failed'
              % (len(FIXTURES), len(FIXTURES) - len(failures), len(failures)))
    return failures


# --------------------------------------------------------------------------
# main
# --------------------------------------------------------------------------

def main(argv=None):
    ap = argparse.ArgumentParser(description='Claim check for WORK-QUEUE.md')
    ap.add_argument('--queue', default=None, help='path to WORK-QUEUE.md')
    ap.add_argument('--json', default=None, help='write the census as JSON here')
    ap.add_argument('--git', action='store_true', help='resolve closure revisions with git')
    ap.add_argument('--validate', action='store_true', help='check against the hand-read answers')
    ap.add_argument('--selftest', action='store_true', help='run the fixture corpus and exit')
    ap.add_argument('--fail-on-stale', action='store_true',
                    help='exit 2 if any opening record is stale')
    ap.add_argument('--explain', type=int, nargs='+', metavar='LINE',
                    help='show how the given line numbers resolve, then exit')
    args = ap.parse_args(argv)

    if args.selftest:
        return 1 if selftest() else 0

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    path = args.queue or os.path.join(root, 'WORK-QUEUE.md')
    try:
        text = open(path, encoding='utf-8').read()
    except OSError as exc:
        print('cannot read queue: %s' % exc, file=sys.stderr)
        return 3

    res = census(text)
    git_dir = os.path.dirname(os.path.abspath(path))

    if args.explain:
        lines = text.splitlines()
        hidx = container_index(lines, loose=False)
        lidx = container_index(lines, loose=True)
        for ln in args.explain:
            if not 1 <= ln <= len(lines):
                print(':%d out of range' % ln)
                continue
            l = lines[ln - 1]
            ids = line_ids(l)
            inh = inherit_id(ln, hidx)
            inl = inherit_id(ln, lidx)
            print(':%d container(strict)=%s container(loose)=%s' % (
                ln, ('Q%s' % inh) if inh else '(none)', ('Q%s' % inl) if inl else '(none)'))
            print('    opening=%s closure=%s ids=%s' %
                  (is_opening_record(l), is_closure_line(l),
                   sorted('Q%d' % i for i in ids) if ids else '(inherit)'))
            print('    %s' % l.strip()[:160])
        return 0

    if args.git:
        for r in res['records']:
            r['revisions'] = annotate_revisions(r, git_dir)

    print('queue %s — %d lines, %s bytes' % (path, res['lines'], format(res['bytes'], ',')))
    print('%-6s %-38s %-7s %-9s %s' % ('line', 'item', 'state', 'closure', 'note'))
    for r in res['records']:
        note = []
        if r['stale']:
            note.append('STALE-opening-line')
        if r['collision']:
            note.append('COLLISION')
        if r['inherited_id']:
            note.append('id-inherited')
        if r['closure_lines_before']:
            note.append('closure-before-opening ' +
                        ','.join(':%d' % x for x in r['closure_lines_before']))
        if len(r['closure_lines_after']) > 1:
            note.append('closures ' + ','.join(':%d' % x for x in r['closure_lines_after']))
        if args.git and r.get('revisions'):
            note.append('revs ' + ','.join(
                '%s:%s' % (v['token'], 'ok' if v['resolves'] else 'NOT-A-GIT-OBJECT')
                for v in r['revisions']))
        print('%-6d %-38s %-7s %-9s %s' % (
            r['line'],
            (r['item'] or '(unnumbered)')[:38],
            r['state'],
            str(r['closure_line'] or '-'),
            ' '.join(note)))

    closed = [r for r in res['records'] if r['state'] == 'CLOSED']
    opened = [r for r in res['records'] if r['state'] == 'OPEN']
    stale = [r for r in res['records'] if r['stale']]
    print('\n%d opening records; %d closed; %d open. %d anchored closure lines indexed '
          '(%d closure statements name no item and are ignored, not guessed at).'
          % (len(res['records']), len(closed), len(opened), res['closure_candidates'],
             len(res['closure_unanchored'])))
    print('open set: %s' % ', '.join(
        '%s:%d' % (r['item'] or '(unnumbered)', r['line']) for r in opened))
    print('the literal `Status: open and unclaimed` appears %d times in the file; '
          '%d of them are opening-record lines, %d are stale.'
          % (res['literal_anywhere'], len(stale), len(stale)))

    rc = 0
    if args.validate:
        bad = []
        for r in res['records']:
            exp = HAND_READ.get(r['line'])
            if exp and exp != r['state']:
                bad.append((r['line'], exp, r['state']))
        n = sum(1 for r in res['records'] if r['line'] in HAND_READ)
        print('hand-read validation: %d/%d match' % (n - len(bad), n))
        for b in bad:
            print('  MISMATCH :%d hand=%s tool=%s' % b)
        if bad:
            rc = 1

    if args.json:
        with open(args.json, 'w', encoding='utf-8') as fh:
            json.dump(res, fh, indent=2, sort_keys=True)
        print('wrote %s' % args.json)

    if args.fail_on_stale and stale:
        rc = 2
    return rc


if __name__ == '__main__':
    sys.exit(main())