diff --git a/tools/queue_claim_check.py b/tools/queue_claim_check.py index 60f7a4b..57e39df 100644 --- a/tools/queue_claim_check.py +++ b/tools/queue_claim_check.py @@ -28,6 +28,22 @@ a line whose first non-space characters are an optional `- `/`* ` bullet followed by `**`. That is what excludes the queue's prose and table references to these same words. +A FENCED BLOCK IS QUOTED MATERIAL, NOT A RECORD (2026-10-19). `queue_index.py` +and `checkin_check.py` already refuse to read structure out of a fence, on the +rule that a quoted heading is not a heading, and `citation_resolve_check.py` +applies the same rule to citations. This file had no fence state, so a tick that +quoted an example record inside a ``` block opened a phantom item: measured on a +19-line fixture carrying one fenced `**Q99 — OPEN …**` and one fenced +`**Status: OPEN …**`, the census printed two OPEN records no writer ever opened +(`before/fence-fixture.txt` in the tick's artifact). The live queue at HEAD has +no record inside a fence — 0 of its fenced lines lead `**` — so the rule moves no +verdict there; it is enforced against the next one. What is set aside is printed +on every run, never silently dropped, and an unclosed fence fails OPEN — nothing +is masked, every line is graded, and `FENCE_UNCLOSED` names the line — because +masking to EOF would hide the newest records, and those are the ones a claim +check exists to surface. The legacy BQ unit is a transcription of another tool +and is left byte-for-byte; its fence delta is printed beside it, not applied. + OPENING RECORD — a record line carrying either (a) `Status:` followed by `open`/`OPEN`, or (b) an item id `Q` followed within 60 characters by an `—`/`–`/`--` @@ -111,7 +127,7 @@ USAGE python3 tools/queue_claim_check.py --queue PATH --json OUT.json python3 tools/queue_claim_check.py --validate # + the hand-read answers python3 tools/queue_claim_check.py --git # resolve closure revisions - python3 tools/queue_claim_check.py --selftest # 17 fixtures, exit 0/1 + python3 tools/queue_claim_check.py --selftest # 20 fixtures, exit 0/1 python3 tools/queue_claim_check.py --census # both closure units + the axis table Exit codes: 0 clean · 1 self-test failure · 2 stale opening line (--fail-on-stale) @@ -179,6 +195,51 @@ def strip_code(line): return CODE_SPAN.sub('', line) +# A FENCED BLOCK IS QUOTED MATERIAL, NOT A RECORD (2026-10-19). `queue_index.py` +# (:207, :266) and `checkin_check.py` (:251) already refuse to read structure out +# of a fence; this scanner had no fence state, so a quoted example record opened a +# phantom item. Line numbers stay absolute -- the mask is a set of 1-based line +# numbers, so every citation into the queue keeps its meaning. +FENCE = re.compile(r'^\s*(?:`{3,}|~{3,})') + + +def fence_mask(lines): + """(fenced 1-based line numbers, block count, opening line if never closed). + + Markdown's rule is that an unclosed fence runs to the end of the document. + For a CLAIM CHECK that rule is the dangerous direction -- masked text cannot + open a record, so a stray fence would hide the newest items, which are + exactly the ones a claim check exists to surface. So an unclosed fence fails + OPEN: nothing is masked, every line is graded, and the opening line is + returned and printed as FENCE_UNCLOSED. Quoted material graded as a record is + a visible row (the operator sees the diagnostic and the row); a record + silently masked is an item nobody sees at all. + """ + out = set() + in_fence = False + opener = None + opened_at = None + blocks = 0 + for n, ln in enumerate(lines, 1): + m = FENCE.match(ln) + if m: + mark = m.group(0).strip()[0] + if not in_fence: + in_fence, opener, opened_at = True, mark, n + blocks += 1 + out.add(n) + continue + if mark == opener: + in_fence, opener = False, None + out.add(n) + continue + if in_fence: + out.add(n) + if in_fence: # malformed document: grade everything + return set(), blocks, opened_at + return out, blocks, None + + def is_record_line(line): return bool(RECORD_PREFIX.match(line)) @@ -337,12 +398,15 @@ def inherit_id(n, hidx): def census(text): lines = text.splitlines() + fenced, n_blocks, unclosed = fence_mask(lines) hidx = container_index(lines, loose=False) lidx = container_index(lines, loose=True) closures = [] # (line, idset, text) unanchored = [] # closure statements that name no item for n, l in enumerate(lines, 1): + if n in fenced: # quoted material, not a statement + continue if not is_closure_line(l): continue ids = line_ids(l) @@ -356,6 +420,8 @@ def census(text): records = [] for n, l in enumerate(lines, 1): + if n in fenced: # quoted material, not a record + continue if not is_opening_record(l): continue lead = lead_item(l) @@ -430,6 +496,9 @@ def census(text): 'closure_candidates': len(closures), 'closure_unanchored': unanchored, 'literal_anywhere': raw_literal, + 'fenced_lines': len(fenced), + 'fenced_blocks': n_blocks, + 'fenced_unclosed': unclosed, } @@ -516,6 +585,26 @@ FIXTURES = [ '## Q40 — second use of the same label\n\n**Status: OPEN — queued.**\n\n' '**Q40 — DONE at 4010b96.**\n', ['Q40:OPEN', 'Q41:OPEN', 'Q40:CLOSED']), + + # The live defect these three cells were written for (2026-10-19): this file + # had no fence state, so a tick quoting an example record inside a fenced + # block opened a phantom item. Pre-fix the first cell reads + # ['Q60:CLOSED', 'Q99:OPEN', 'Q99:CLOSED'] and the second reads two Q61 + # records (the fenced `**Status:` inherited the header, then collided). + ('fence: a record inside a ``` block is quoted material, not a record', + '## Q60 — section\n\n**Q60 — OPEN (lane).**\n\n**Q60 — DONE at 4010b96.**\n\n' + 'Prose quoting a tick:\n\n```\n**Q99 — OPEN (pipe).**\n' + '**Q99 — DONE at 4010b96.**\n```\n', + ['Q60:CLOSED']), + + ('fence: a tilde fence is a fence, and a fenced Status line inherits nothing', + '## Q61 — section\n\n**Q61 — OPEN (lane).**\n\n~~~\n' + '**Status: OPEN — queued.**\n~~~\n', + ['Q61:OPEN']), + + ('fence: an unclosed fence fails open — every line graded, the line named', + '## Q62 — section\n\n**Q62 — OPEN (lane).**\n\n```\n**Q63 — OPEN (lane).**\n', + ['Q62:OPEN', 'Q63:OPEN']), ] UNIT_FIXTURES = [ @@ -541,26 +630,36 @@ STALE_EXPECT = { 'a bold prose record naming another item does not steal the container': 1, } +# The unclosed-fence cell pins the DIAGNOSTIC, not just the masking: the value is +# the line the fence opened on, which the report must print. +UNCLOSED_EXPECT = { + 'fence: an unclosed fence fails open — every line graded, the line named': 5, +} + def fixture_signature(text): res = census(text) sig = ['%s:%s' % (r['item'] or 'NONE', r['state']) for r in res['records']] - return sig, len([r for r in res['records'] if r['stale']]) + return sig, len([r for r in res['records'] if r['stale']]), res['fenced_unclosed'] def selftest(verbose=True): failures = [] for name, text, expect in FIXTURES: - got, stale = fixture_signature(text) + got, stale, unclosed = fixture_signature(text) ok = got == expect want_stale = STALE_EXPECT.get(name, 0) if stale != want_stale: ok = False + want_unclosed = UNCLOSED_EXPECT.get(name) + if want_unclosed is not None and unclosed != want_unclosed: + ok = False if verbose: print('%s %s' % ('PASS' if ok else 'FAIL', name)) if not ok: - print(' expected records %s (stale %d)\n got records %s (stale %d)' - % (expect, want_stale, got, stale)) + print(' expected records %s (stale %d, unclosed %s)\n' + ' got records %s (stale %d, unclosed %s)' + % (expect, want_stale, want_unclosed, got, stale, unclosed)) if not ok: failures.append(name) @@ -659,6 +758,7 @@ def main(argv=None): if args.explain: lines = text.splitlines() + fenced, _b, _u = fence_mask(lines) hidx = container_index(lines, loose=False) lidx = container_index(lines, loose=True) for ln in args.explain: @@ -671,8 +771,8 @@ def main(argv=None): inl = inherit_id(ln, lidx) print(':%d container(strict)=%s container(loose)=%s' % ( ln, ('Q%s' % inh) if inh else '(none)', ('Q%s' % inl) if inl else '(none)')) - print(' opening=%s closure=%s ids=%s' % - (is_opening_record(l), is_closure_line(l), + print(' opening=%s closure=%s fenced=%s ids=%s' % + (is_opening_record(l), is_closure_line(l), ln in fenced, sorted('Q%d' % i for i in ids) if ids else '(inherit)')) print(' %s' % l.strip()[:160]) return 0 @@ -682,6 +782,12 @@ def main(argv=None): r['revisions'] = annotate_revisions(r, git_dir) print('queue %s — %d lines, %s bytes' % (path, res['lines'], format(res['bytes'], ','))) + print('fenced skipped: %d line(s) in %d block(s) — a fenced block is quoted ' + 'material, not a record' % (res['fenced_lines'], res['fenced_blocks'])) + if res['fenced_unclosed']: + print('FENCE_UNCLOSED at :%d — the fence never closes, so NOTHING is masked: ' + 'every line below it is graded, and quoted material may appear as a record ' + 'rather than a record disappearing' % res['fenced_unclosed']) print('%-6s %-38s %-7s %-9s %s' % ('line', 'item', 'state', 'closure', 'note')) for r in res['records']: note = [] @@ -733,17 +839,21 @@ def main(argv=None): if args.census: lines = text.splitlines() - acs = {n for n, l in enumerate(lines, 1) if is_closure_line(l)} + fenced, _blocks, _unclosed = fence_mask(lines) + acs = {n for n, l in enumerate(lines, 1) if n not in fenced and is_closure_line(l)} bq = set(legacy_closures(text)) print('\n--- closure-unit census (cite a number only with its unit) ---') print('ACS anchored closure statement (this file) = %d' % len(acs)) print(' record line leading `**Q`/`**Status:`, closure word in the first') print(' %d chars, code spans stripped, exclusions tested at the lead,' % CLOSURE_WINDOW) - print(' anchored to its own id or the nearest preceding container.') + print(' anchored to its own id or the nearest preceding container; fenced') + print(' lines excluded (%d set aside).' % len(fenced)) print('BQ column-0 bold closure line naming a Q id (legacy q66_census6.py:54-62) = %d' % len(bq)) print(' line starting at column 0 with `**`, a Q token anywhere, a') print(' closure word anywhere and case-insensitively, no legacy exclusion phrase.') + print(' Transcribed byte-for-byte and NOT fenced: %d of its lines stand inside a ' + 'fence here, a delta printed and not applied.' % len(bq & fenced)) print('overlap A&B = %d ; ACS-only = %d ; BQ-only = %d ; symmetric difference = %d' % (len(acs & bq), len(acs - bq), len(bq - acs), len(acs ^ bq))) print('NOT a unit conversion: the difference between the two headline numbers is')