346 lines
14 KiB
Python
346 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""q114_verify.py -- grade the 2026-10-31 free-tier audit against Q114's done-condition.
|
|
|
|
WHY THIS EXISTS
|
|
Q114: `wizard-work/quill/free-tier-ai-stack-audit.md` presented twelve providers
|
|
under a "Verified Free-Tier Providers" heading, sourced from six aggregator
|
|
pages, with no per-row URL, no fetch date, no expiry column and no eligibility
|
|
verdict answering `FREE-TIER-CANDIDATES.md`'s 2026-10-06 rule ("if a free tier
|
|
is depleted and continuing requires payment, the service is ineligible -- drop
|
|
it, don't fund it"). Three of the twelve rows were self-terminating grants.
|
|
|
|
The repair rewrote the document as sixteen rows measured on one date:
|
|
every row names a source id, a fetch date, an `Expires` cell, an `Eligible`
|
|
verdict and a `Basis` word; a source ledger carries the URL, HTTP status, byte
|
|
count and sha256 of the bytes each quote was taken from; and the run's
|
|
transcript is committed beside the document as `q114-fetch-2026-10-31.tsv`.
|
|
|
|
WHAT IT GRADES
|
|
document the committed pair (document, transcript) satisfies every structural
|
|
clause of the done-condition: 16 rows, one source id each, all dated
|
|
2026-10-31, a non-empty Expires cell, Eligible in {Y,N,unknown},
|
|
Basis in a closed vocabulary, a quoted phrase on every documented
|
|
row, 17 ledger rows whose digests match the transcript, and no
|
|
"Verified" heading anywhere.
|
|
mutants five edits of a copy -- an `Eligible` cell set to `maybe`, a fetch
|
|
date moved to 2025-10-28, a ledger digest truncated, a ledger row
|
|
deleted, a data row deleted -- each must produce a named fault. A
|
|
gate that cannot go red on these is the class Q117 filed.
|
|
live (opt-in, --live) re-fetches every ledger URL, re-hashes the body and
|
|
checks each documented row's quote is still on the page; reports
|
|
which digests moved. rc 2 when the network is unavailable, because
|
|
a live re-read that did not happen is not a pass.
|
|
|
|
USAGE
|
|
python3 wizard-work/volt/q114_verify.py [--repo DIR] [--live] [--verbose]
|
|
EXIT
|
|
0 every leg as expected; 1 a graded fault; 2 could not grade (no document, no
|
|
transcript, no python3, or --live with no network).
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
DOC = "wizard-work/quill/free-tier-ai-stack-audit.md"
|
|
TSV = "wizard-work/quill/q114-fetch-2026-10-31.tsv"
|
|
ROWS_EXPECTED = 16
|
|
LEDGER_EXPECTED = 17
|
|
DATE = "2026-10-31"
|
|
ELIGIBLE = ("Y", "N", "unknown")
|
|
BASIS = ("documented", "derived", "contradicted", "not-stated", "unmeasured")
|
|
EXPIRES_OK = re.compile(r"^(none stated|not stated|30 days after activation.*)$")
|
|
SHA = re.compile(r"^[0-9a-f]{64}$")
|
|
LEDGER_ID = re.compile(r"^s\d{2}$")
|
|
STAMP = re.compile(r"^2026-10-31T\d\d:\d\d:\d\dZ$")
|
|
DEFAULT_REPO = os.path.dirname(os.path.dirname(os.path.dirname(
|
|
os.path.abspath(__file__))))
|
|
|
|
|
|
def cells(line):
|
|
return [c.strip() for c in line.strip().strip("|").split("|")]
|
|
|
|
|
|
def section(lines, heading):
|
|
"""Rows of the first markdown table after `heading`, until a blank line after it."""
|
|
out, seen, started = [], False, False
|
|
for line in lines:
|
|
if line.startswith(heading):
|
|
seen = True
|
|
continue
|
|
if not seen:
|
|
continue
|
|
if line.startswith("|"):
|
|
out.append(line)
|
|
started = True
|
|
elif started:
|
|
break
|
|
return out
|
|
|
|
|
|
def parse_table(block, id_re):
|
|
"""Return [(id, cells)] for data rows whose first cell matches id_re."""
|
|
rows = []
|
|
for line in block:
|
|
c = cells(line)
|
|
if c and id_re.match(c[0]):
|
|
rows.append((c[0], c))
|
|
return rows
|
|
|
|
|
|
def grade(doc, tsv):
|
|
"""Pure reading: return a list of fault strings (empty means the pair is sound)."""
|
|
faults = []
|
|
lines = doc.splitlines()
|
|
|
|
rows = parse_table(section(lines, "## Rows"), re.compile(r"^\d+$"))
|
|
ledger = parse_table(section(lines, "## Source ledger"), LEDGER_ID)
|
|
|
|
if len(rows) != ROWS_EXPECTED:
|
|
faults.append("rows: %d data row(s), expected %d" % (len(rows), ROWS_EXPECTED))
|
|
if len(ledger) != LEDGER_EXPECTED:
|
|
faults.append("ledger: %d row(s), expected %d" % (len(ledger), LEDGER_EXPECTED))
|
|
|
|
header = None
|
|
for line in section(lines, "## Rows"):
|
|
c = cells(line)
|
|
if c and c[0] == "#":
|
|
header = c
|
|
break
|
|
for need in ("Src", "Fetched", "Expires", "Eligible", "Basis"):
|
|
if not header or need not in header:
|
|
faults.append("rows: table header has no %r column" % need)
|
|
|
|
ledger_ids = {i for i, _ in ledger}
|
|
for want, (rid, c) in enumerate(rows, start=1):
|
|
if rid != str(want):
|
|
faults.append("rows: id %s where %d was expected (order)" % (rid, want))
|
|
if len(c) != 9:
|
|
faults.append("rows: id %s has %d cell(s), expected 9" % (rid, len(c)))
|
|
continue
|
|
_, provider, src, fetched, tier, models, expires, eligible, basis = c
|
|
if not provider:
|
|
faults.append("rows: id %s has no provider" % rid)
|
|
if not LEDGER_ID.match(src) or src not in ledger_ids:
|
|
faults.append("rows: id %s source %r is not in the ledger" % (rid, src))
|
|
if fetched != DATE:
|
|
faults.append("rows: id %s fetched %r, expected %s" % (rid, fetched, DATE))
|
|
if not tier:
|
|
faults.append("rows: id %s has an empty free-tier cell" % rid)
|
|
if not models:
|
|
faults.append("rows: id %s has an empty models cell" % rid)
|
|
if not EXPIRES_OK.match(expires):
|
|
faults.append("rows: id %s expiry %r is not a stated expiry" % (rid, expires))
|
|
if eligible not in ELIGIBLE:
|
|
faults.append("rows: id %s eligibility %r not in %s"
|
|
% (rid, eligible, "/".join(ELIGIBLE)))
|
|
if basis not in BASIS:
|
|
faults.append("rows: id %s basis %r not in %s"
|
|
% (rid, basis, "/".join(BASIS)))
|
|
if basis in ("documented", "contradicted"):
|
|
quotes = re.findall(r'"([^"]{15,})"', tier)
|
|
if not quotes:
|
|
faults.append("rows: id %s is %s but quotes no phrase" % (rid, basis))
|
|
|
|
for want, (lid, c) in enumerate(ledger, start=1):
|
|
if lid != "s%02d" % want:
|
|
faults.append("ledger: id %s where s%02d was expected (order)" % (lid, want))
|
|
if len(c) != 7:
|
|
faults.append("ledger: id %s has %d cell(s), expected 7" % (lid, len(c)))
|
|
continue
|
|
_, slug, url, http, size, sha, stamp = c
|
|
if not re.match(r"^https://", url):
|
|
faults.append("ledger: id %s url %r is not an https URL" % (lid, url))
|
|
if http != "200":
|
|
faults.append("ledger: id %s http %r, expected 200" % (lid, http))
|
|
if not size.isdigit() or int(size) < 100:
|
|
faults.append("ledger: id %s byte count %r is not a positive size"
|
|
% (lid, size))
|
|
if not SHA.match(sha):
|
|
faults.append("ledger: id %s digest %r is not 64 hex" % (lid, sha[:20]))
|
|
if not STAMP.match(stamp):
|
|
faults.append("ledger: id %s fetched %r is not a %s stamp"
|
|
% (lid, stamp, DATE))
|
|
|
|
tsv_lines = [l for l in tsv.splitlines() if l.strip()]
|
|
if not tsv_lines or tsv_lines[0].split("\t") != [
|
|
"slug", "url", "http", "bytes", "sha256", "fetched_utc"]:
|
|
faults.append("transcript: header is not the six committed columns")
|
|
records = []
|
|
else:
|
|
records = [l.split("\t") for l in tsv_lines[1:]]
|
|
if len(records) < LEDGER_EXPECTED:
|
|
faults.append("transcript: %d row(s), expected at least %d"
|
|
% (len(records), LEDGER_EXPECTED))
|
|
index = {}
|
|
for rec in records:
|
|
if len(rec) != 6:
|
|
faults.append("transcript: a row has %d column(s)" % len(rec))
|
|
continue
|
|
index[rec[0]] = rec
|
|
for lid, c in ledger:
|
|
if len(c) != 7:
|
|
continue
|
|
slug, url, sha = c[1], c[2], c[5]
|
|
rec = index.get(slug)
|
|
if rec is None:
|
|
faults.append("ledger: id %s slug %r is absent from the transcript"
|
|
% (lid, slug))
|
|
continue
|
|
if rec[1] != url or rec[4] != sha or rec[3] != c[4] or rec[2] != c[3]:
|
|
faults.append("ledger: id %s disagrees with the transcript for %s"
|
|
% (lid, slug))
|
|
|
|
if "FREE-TIER-CANDIDATES.md" not in doc:
|
|
faults.append("rule: the document does not cite FREE-TIER-CANDIDATES.md")
|
|
if "drop it, don't fund it" not in doc:
|
|
faults.append("rule: the 2026-10-06 eligibility sentence is not quoted")
|
|
if "Verified Free-Tier" in doc:
|
|
faults.append("rule: the 'Verified Free-Tier' heading survives")
|
|
for line in lines:
|
|
if line.startswith("##") and re.search(r"\bVerified\b", line):
|
|
faults.append("rule: heading still claims verification: %r" % line[:60])
|
|
break
|
|
return faults
|
|
|
|
|
|
def mutate(doc, kind):
|
|
"""Apply one seeded edit; return (mutated_text, expected fault substring)."""
|
|
if kind == "eligible":
|
|
out = []
|
|
for line in doc.splitlines():
|
|
c = cells(line)
|
|
if c and c[0] == "3" and len(c) == 9 and c[7] == "Y":
|
|
line = line.replace("| Y | derived |", "| maybe | derived |")
|
|
out.append(line)
|
|
return "\n".join(out) + "\n", "eligibility 'maybe'"
|
|
if kind == "dated":
|
|
return doc.replace("| 2026-10-31 |", "| 2025-10-28 |", 1), "fetched '2025-10-28'"
|
|
if kind == "digest":
|
|
return re.sub(r"\| ([0-9a-f]{64}) \| 2026-10-31T07:13:25Z",
|
|
lambda m: "| " + m.group(1)[:12] + " | 2026-10-31T07:13:25Z",
|
|
doc, count=1), "digest"
|
|
if kind == "ledger-row":
|
|
out = re.sub(r"^\| s14 \|.*\n", "", doc, count=1, flags=re.M)
|
|
return out, "is not in the ledger"
|
|
if kind == "data-row":
|
|
return re.sub(r"^\| 13 \| Pollinations \|.*\n", "", doc, count=1, flags=re.M), \
|
|
"data row(s)"
|
|
raise SystemExit("unknown mutation " + kind)
|
|
|
|
|
|
def leg_document(repo):
|
|
doc_p, tsv_p = os.path.join(repo, DOC), os.path.join(repo, TSV)
|
|
if not os.path.isfile(doc_p):
|
|
return "FAULT", "no %s" % DOC
|
|
if not os.path.isfile(tsv_p):
|
|
return "FAULT", "no %s" % TSV
|
|
doc = open(doc_p, encoding="utf-8").read()
|
|
tsv = open(tsv_p, encoding="utf-8").read()
|
|
faults = grade(doc, tsv)
|
|
if faults:
|
|
return "FAULT", "%d fault(s): %s" % (len(faults), " | ".join(faults[:4]))
|
|
return "OK", "%d rows, %d ledger rows, transcript agrees, rule cited" % (
|
|
ROWS_EXPECTED, LEDGER_EXPECTED)
|
|
|
|
|
|
def leg_mutants(repo):
|
|
doc = open(os.path.join(repo, DOC), encoding="utf-8").read()
|
|
tsv = open(os.path.join(repo, TSV), encoding="utf-8").read()
|
|
if grade(doc, tsv):
|
|
return "FAULT", "control copy is already red -- mutants would prove nothing"
|
|
caught, misses = 0, []
|
|
for kind in ("eligible", "dated", "digest", "ledger-row", "data-row"):
|
|
mutated, expect = mutate(doc, kind)
|
|
if mutated == doc:
|
|
misses.append("%s: mutation did not change the document" % kind)
|
|
continue
|
|
found = grade(mutated, tsv)
|
|
named = [f for f in found if expect in f]
|
|
if named:
|
|
caught += 1
|
|
else:
|
|
misses.append("%s: expected a fault naming %r, got %r"
|
|
% (kind, expect, found[:1]))
|
|
if misses:
|
|
return "FAULT", "; ".join(misses)
|
|
return "OK", "%d/%d seeded edits each red a named cell" % (caught, 5)
|
|
|
|
|
|
def leg_live(repo, verbose=False):
|
|
"""Opt-in: re-fetch each ledger URL and check the digests and quotes still hold."""
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
doc = open(os.path.join(repo, DOC), encoding="utf-8").read()
|
|
rows = parse_table(section(doc.splitlines(), "## Rows"), re.compile(r"^\d+$"))
|
|
ledger = parse_table(section(doc.splitlines(), "## Source ledger"), LEDGER_ID)
|
|
by_slug = {c[1]: c for _, c in ledger}
|
|
phrases = {}
|
|
for rid, c in rows:
|
|
if len(c) == 9 and c[8] in ("documented", "contradicted"):
|
|
q = re.findall(r'"([^"]{15,})"', c[4])
|
|
phrases[by_slug.get(c[2], [None, None])[1]] = q[0] if q else None
|
|
|
|
moved, unreachable, checked = [], [], 0
|
|
for lid, c in ledger:
|
|
slug, url, sha = c[1], c[2], c[5]
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "volt-q114/1"})
|
|
body = urllib.request.urlopen(req, timeout=45).read()
|
|
except Exception as exc: # noqa: BLE001
|
|
unreachable.append("%s (%s)" % (slug, type(exc).__name__))
|
|
continue
|
|
checked += 1
|
|
if hashlib.sha256(body).hexdigest() != sha:
|
|
moved.append(slug)
|
|
want = phrases.get(slug)
|
|
if want:
|
|
text = re.sub(r"\s+", " ", body.decode("utf-8", "replace"))
|
|
if re.sub(r"\s+", " ", want) not in text:
|
|
moved.append(slug + "(quote)")
|
|
if verbose:
|
|
print(" live: checked %d/%d url(s); unreachable %s"
|
|
% (checked, len(ledger), unreachable or "none"))
|
|
if checked == 0:
|
|
return "FAULT", "no URL could be re-read -- a live leg that fetched nothing is not a pass"
|
|
if moved:
|
|
return "OK", "%d/%d re-read, drifted: %s" % (checked, len(ledger), ", ".join(moved[:6]))
|
|
return "OK", "%d/%d re-read, all digests and quotes unchanged" % (checked, len(ledger))
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--repo", default=DEFAULT_REPO)
|
|
ap.add_argument("--live", action="store_true")
|
|
ap.add_argument("--verbose", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
repo = os.path.abspath(a.repo)
|
|
if not os.path.isdir(repo):
|
|
print("q114: could not grade -- no such directory: " + repo)
|
|
return 2
|
|
if not os.path.isfile(os.path.join(repo, DOC)):
|
|
print("q114: could not grade -- no %s under %s" % (DOC, repo))
|
|
return 2
|
|
|
|
legs = [("document", leg_document), ("mutants", leg_mutants)]
|
|
if a.live:
|
|
legs.append(("live", lambda r: leg_live(r, a.verbose)))
|
|
faults = 0
|
|
print("q114: %s" % repo)
|
|
for name, fn in legs:
|
|
verdict, detail = fn(repo)
|
|
if verdict != "OK":
|
|
faults += 1
|
|
print(" %-9s %-5s %s" % (name, verdict, detail))
|
|
print("q114: %d leg(s), %d passed, %d fault(s)"
|
|
% (len(legs), len(legs) - faults, faults))
|
|
return 1 if faults else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|