volt: q114 -- free-tier audit re-read from 17 provider pages, graded by CHECK-q114
This commit is contained in:
volt committed
1 parent
c0a22cbd32
commit
7d5de199e3
7 files changed
+549
-46
No files matched your search
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Q114 CHECK -- the free-tier audit must carry, for every row, a primary URL
|
||||
# with a fetch date, an expiry cell and an eligibility verdict answering
|
||||
# FREE-TIER-CANDIDATES.md's 2026-10-06 rule, and no "Verified" claim.
|
||||
#
|
||||
# rc 0 = every leg as expected; 1 = a graded fault; 2 = could not grade.
|
||||
# The work is done by wizard-work/volt/q114_verify.py beside this file: it
|
||||
# grades the committed document against the committed fetch transcript
|
||||
# (q114-fetch-2026-10-31.tsv), then replays five seeded edits of a copy and
|
||||
# requires each to red a named cell. `--live` re-fetches every source URL and
|
||||
# reports which digests moved; it is opt-in because a live re-read that did not
|
||||
# happen must not read as a pass.
|
||||
set -u
|
||||
REPO="$(cd "$(dirname "$0")/../.." && pwd)" || exit 2
|
||||
if ! cd "$REPO"; then
|
||||
echo "q114: could not grade -- cannot cd to $REPO"
|
||||
exit 2
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "q114: could not grade -- no python3 on PATH"
|
||||
exit 2
|
||||
fi
|
||||
exec python3 wizard-work/volt/q114_verify.py "$@"
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env python3
|
||||
"""q114_verify.py -- grade the 2026-10-31 free-tier audit against Q114's done-condition.
|
||||
|
||||
WHY THIS EXISTS
|
||||
Q114: `wizard-work/quill/free-tier-ai-stack-audit.md` presented twelve providers
|
||||
under a "Verified Free-Tier Providers" heading, sourced from six aggregator
|
||||
pages, with no per-row URL, no fetch date, no expiry column and no eligibility
|
||||
verdict answering `FREE-TIER-CANDIDATES.md`'s 2026-10-06 rule ("if a free tier
|
||||
is depleted and continuing requires payment, the service is ineligible -- drop
|
||||
it, don't fund it"). Three of the twelve rows were self-terminating grants.
|
||||
|
||||
The repair rewrote the document as sixteen rows measured on one date:
|
||||
every row names a source id, a fetch date, an `Expires` cell, an `Eligible`
|
||||
verdict and a `Basis` word; a source ledger carries the URL, HTTP status, byte
|
||||
count and sha256 of the bytes each quote was taken from; and the run's
|
||||
transcript is committed beside the document as `q114-fetch-2026-10-31.tsv`.
|
||||
|
||||
WHAT IT GRADES
|
||||
document the committed pair (document, transcript) satisfies every structural
|
||||
clause of the done-condition: 16 rows, one source id each, all dated
|
||||
2026-10-31, a non-empty Expires cell, Eligible in {Y,N,unknown},
|
||||
Basis in a closed vocabulary, a quoted phrase on every documented
|
||||
row, 17 ledger rows whose digests match the transcript, and no
|
||||
"Verified" heading anywhere.
|
||||
mutants five edits of a copy -- an `Eligible` cell set to `maybe`, a fetch
|
||||
date moved to 2025-10-28, a ledger digest truncated, a ledger row
|
||||
deleted, a data row deleted -- each must produce a named fault. A
|
||||
gate that cannot go red on these is the class Q117 filed.
|
||||
live (opt-in, --live) re-fetches every ledger URL, re-hashes the body and
|
||||
checks each documented row's quote is still on the page; reports
|
||||
which digests moved. rc 2 when the network is unavailable, because
|
||||
a live re-read that did not happen is not a pass.
|
||||
|
||||
USAGE
|
||||
python3 wizard-work/volt/q114_verify.py [--repo DIR] [--live] [--verbose]
|
||||
EXIT
|
||||
0 every leg as expected; 1 a graded fault; 2 could not grade (no document, no
|
||||
transcript, no python3, or --live with no network).
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
DOC = "wizard-work/quill/free-tier-ai-stack-audit.md"
|
||||
TSV = "wizard-work/quill/q114-fetch-2026-10-31.tsv"
|
||||
ROWS_EXPECTED = 16
|
||||
LEDGER_EXPECTED = 17
|
||||
DATE = "2026-10-31"
|
||||
ELIGIBLE = ("Y", "N", "unknown")
|
||||
BASIS = ("documented", "derived", "contradicted", "not-stated", "unmeasured")
|
||||
EXPIRES_OK = re.compile(r"^(none stated|not stated|30 days after activation.*)$")
|
||||
SHA = re.compile(r"^[0-9a-f]{64}$")
|
||||
LEDGER_ID = re.compile(r"^s\d{2}$")
|
||||
STAMP = re.compile(r"^2026-10-31T\d\d:\d\d:\d\dZ$")
|
||||
DEFAULT_REPO = os.path.dirname(os.path.dirname(os.path.dirname(
|
||||
os.path.abspath(__file__))))
|
||||
|
||||
|
||||
def cells(line):
|
||||
return [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
|
||||
|
||||
def section(lines, heading):
|
||||
"""Rows of the first markdown table after `heading`, until a blank line after it."""
|
||||
out, seen, started = [], False, False
|
||||
for line in lines:
|
||||
if line.startswith(heading):
|
||||
seen = True
|
||||
continue
|
||||
if not seen:
|
||||
continue
|
||||
if line.startswith("|"):
|
||||
out.append(line)
|
||||
started = True
|
||||
elif started:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def parse_table(block, id_re):
|
||||
"""Return [(id, cells)] for data rows whose first cell matches id_re."""
|
||||
rows = []
|
||||
for line in block:
|
||||
c = cells(line)
|
||||
if c and id_re.match(c[0]):
|
||||
rows.append((c[0], c))
|
||||
return rows
|
||||
|
||||
|
||||
def grade(doc, tsv):
|
||||
"""Pure reading: return a list of fault strings (empty means the pair is sound)."""
|
||||
faults = []
|
||||
lines = doc.splitlines()
|
||||
|
||||
rows = parse_table(section(lines, "## Rows"), re.compile(r"^\d+$"))
|
||||
ledger = parse_table(section(lines, "## Source ledger"), LEDGER_ID)
|
||||
|
||||
if len(rows) != ROWS_EXPECTED:
|
||||
faults.append("rows: %d data row(s), expected %d" % (len(rows), ROWS_EXPECTED))
|
||||
if len(ledger) != LEDGER_EXPECTED:
|
||||
faults.append("ledger: %d row(s), expected %d" % (len(ledger), LEDGER_EXPECTED))
|
||||
|
||||
header = None
|
||||
for line in section(lines, "## Rows"):
|
||||
c = cells(line)
|
||||
if c and c[0] == "#":
|
||||
header = c
|
||||
break
|
||||
for need in ("Src", "Fetched", "Expires", "Eligible", "Basis"):
|
||||
if not header or need not in header:
|
||||
faults.append("rows: table header has no %r column" % need)
|
||||
|
||||
ledger_ids = {i for i, _ in ledger}
|
||||
for want, (rid, c) in enumerate(rows, start=1):
|
||||
if rid != str(want):
|
||||
faults.append("rows: id %s where %d was expected (order)" % (rid, want))
|
||||
if len(c) != 9:
|
||||
faults.append("rows: id %s has %d cell(s), expected 9" % (rid, len(c)))
|
||||
continue
|
||||
_, provider, src, fetched, tier, models, expires, eligible, basis = c
|
||||
if not provider:
|
||||
faults.append("rows: id %s has no provider" % rid)
|
||||
if not LEDGER_ID.match(src) or src not in ledger_ids:
|
||||
faults.append("rows: id %s source %r is not in the ledger" % (rid, src))
|
||||
if fetched != DATE:
|
||||
faults.append("rows: id %s fetched %r, expected %s" % (rid, fetched, DATE))
|
||||
if not tier:
|
||||
faults.append("rows: id %s has an empty free-tier cell" % rid)
|
||||
if not models:
|
||||
faults.append("rows: id %s has an empty models cell" % rid)
|
||||
if not EXPIRES_OK.match(expires):
|
||||
faults.append("rows: id %s expiry %r is not a stated expiry" % (rid, expires))
|
||||
if eligible not in ELIGIBLE:
|
||||
faults.append("rows: id %s eligibility %r not in %s"
|
||||
% (rid, eligible, "/".join(ELIGIBLE)))
|
||||
if basis not in BASIS:
|
||||
faults.append("rows: id %s basis %r not in %s"
|
||||
% (rid, basis, "/".join(BASIS)))
|
||||
if basis in ("documented", "contradicted"):
|
||||
quotes = re.findall(r'"([^"]{15,})"', tier)
|
||||
if not quotes:
|
||||
faults.append("rows: id %s is %s but quotes no phrase" % (rid, basis))
|
||||
|
||||
for want, (lid, c) in enumerate(ledger, start=1):
|
||||
if lid != "s%02d" % want:
|
||||
faults.append("ledger: id %s where s%02d was expected (order)" % (lid, want))
|
||||
if len(c) != 7:
|
||||
faults.append("ledger: id %s has %d cell(s), expected 7" % (lid, len(c)))
|
||||
continue
|
||||
_, slug, url, http, size, sha, stamp = c
|
||||
if not re.match(r"^https://", url):
|
||||
faults.append("ledger: id %s url %r is not an https URL" % (lid, url))
|
||||
if http != "200":
|
||||
faults.append("ledger: id %s http %r, expected 200" % (lid, http))
|
||||
if not size.isdigit() or int(size) < 100:
|
||||
faults.append("ledger: id %s byte count %r is not a positive size"
|
||||
% (lid, size))
|
||||
if not SHA.match(sha):
|
||||
faults.append("ledger: id %s digest %r is not 64 hex" % (lid, sha[:20]))
|
||||
if not STAMP.match(stamp):
|
||||
faults.append("ledger: id %s fetched %r is not a %s stamp"
|
||||
% (lid, stamp, DATE))
|
||||
|
||||
tsv_lines = [l for l in tsv.splitlines() if l.strip()]
|
||||
if not tsv_lines or tsv_lines[0].split("\t") != [
|
||||
"slug", "url", "http", "bytes", "sha256", "fetched_utc"]:
|
||||
faults.append("transcript: header is not the six committed columns")
|
||||
records = []
|
||||
else:
|
||||
records = [l.split("\t") for l in tsv_lines[1:]]
|
||||
if len(records) < LEDGER_EXPECTED:
|
||||
faults.append("transcript: %d row(s), expected at least %d"
|
||||
% (len(records), LEDGER_EXPECTED))
|
||||
index = {}
|
||||
for rec in records:
|
||||
if len(rec) != 6:
|
||||
faults.append("transcript: a row has %d column(s)" % len(rec))
|
||||
continue
|
||||
index[rec[0]] = rec
|
||||
for lid, c in ledger:
|
||||
if len(c) != 7:
|
||||
continue
|
||||
slug, url, sha = c[1], c[2], c[5]
|
||||
rec = index.get(slug)
|
||||
if rec is None:
|
||||
faults.append("ledger: id %s slug %r is absent from the transcript"
|
||||
% (lid, slug))
|
||||
continue
|
||||
if rec[1] != url or rec[4] != sha or rec[3] != c[4] or rec[2] != c[3]:
|
||||
faults.append("ledger: id %s disagrees with the transcript for %s"
|
||||
% (lid, slug))
|
||||
|
||||
if "FREE-TIER-CANDIDATES.md" not in doc:
|
||||
faults.append("rule: the document does not cite FREE-TIER-CANDIDATES.md")
|
||||
if "drop it, don't fund it" not in doc:
|
||||
faults.append("rule: the 2026-10-06 eligibility sentence is not quoted")
|
||||
if "Verified Free-Tier" in doc:
|
||||
faults.append("rule: the 'Verified Free-Tier' heading survives")
|
||||
for line in lines:
|
||||
if line.startswith("##") and re.search(r"\bVerified\b", line):
|
||||
faults.append("rule: heading still claims verification: %r" % line[:60])
|
||||
break
|
||||
return faults
|
||||
|
||||
|
||||
def mutate(doc, kind):
|
||||
"""Apply one seeded edit; return (mutated_text, expected fault substring)."""
|
||||
if kind == "eligible":
|
||||
out = []
|
||||
for line in doc.splitlines():
|
||||
c = cells(line)
|
||||
if c and c[0] == "3" and len(c) == 9 and c[7] == "Y":
|
||||
line = line.replace("| Y | derived |", "| maybe | derived |")
|
||||
out.append(line)
|
||||
return "\n".join(out) + "\n", "eligibility 'maybe'"
|
||||
if kind == "dated":
|
||||
return doc.replace("| 2026-10-31 |", "| 2025-10-28 |", 1), "fetched '2025-10-28'"
|
||||
if kind == "digest":
|
||||
return re.sub(r"\| ([0-9a-f]{64}) \| 2026-10-31T07:13:25Z",
|
||||
lambda m: "| " + m.group(1)[:12] + " | 2026-10-31T07:13:25Z",
|
||||
doc, count=1), "digest"
|
||||
if kind == "ledger-row":
|
||||
out = re.sub(r"^\| s14 \|.*\n", "", doc, count=1, flags=re.M)
|
||||
return out, "is not in the ledger"
|
||||
if kind == "data-row":
|
||||
return re.sub(r"^\| 13 \| Pollinations \|.*\n", "", doc, count=1, flags=re.M), \
|
||||
"data row(s)"
|
||||
raise SystemExit("unknown mutation " + kind)
|
||||
|
||||
|
||||
def leg_document(repo):
|
||||
doc_p, tsv_p = os.path.join(repo, DOC), os.path.join(repo, TSV)
|
||||
if not os.path.isfile(doc_p):
|
||||
return "FAULT", "no %s" % DOC
|
||||
if not os.path.isfile(tsv_p):
|
||||
return "FAULT", "no %s" % TSV
|
||||
doc = open(doc_p, encoding="utf-8").read()
|
||||
tsv = open(tsv_p, encoding="utf-8").read()
|
||||
faults = grade(doc, tsv)
|
||||
if faults:
|
||||
return "FAULT", "%d fault(s): %s" % (len(faults), " | ".join(faults[:4]))
|
||||
return "OK", "%d rows, %d ledger rows, transcript agrees, rule cited" % (
|
||||
ROWS_EXPECTED, LEDGER_EXPECTED)
|
||||
|
||||
|
||||
def leg_mutants(repo):
|
||||
doc = open(os.path.join(repo, DOC), encoding="utf-8").read()
|
||||
tsv = open(os.path.join(repo, TSV), encoding="utf-8").read()
|
||||
if grade(doc, tsv):
|
||||
return "FAULT", "control copy is already red -- mutants would prove nothing"
|
||||
caught, misses = 0, []
|
||||
for kind in ("eligible", "dated", "digest", "ledger-row", "data-row"):
|
||||
mutated, expect = mutate(doc, kind)
|
||||
if mutated == doc:
|
||||
misses.append("%s: mutation did not change the document" % kind)
|
||||
continue
|
||||
found = grade(mutated, tsv)
|
||||
named = [f for f in found if expect in f]
|
||||
if named:
|
||||
caught += 1
|
||||
else:
|
||||
misses.append("%s: expected a fault naming %r, got %r"
|
||||
% (kind, expect, found[:1]))
|
||||
if misses:
|
||||
return "FAULT", "; ".join(misses)
|
||||
return "OK", "%d/%d seeded edits each red a named cell" % (caught, 5)
|
||||
|
||||
|
||||
def leg_live(repo, verbose=False):
|
||||
"""Opt-in: re-fetch each ledger URL and check the digests and quotes still hold."""
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
doc = open(os.path.join(repo, DOC), encoding="utf-8").read()
|
||||
rows = parse_table(section(doc.splitlines(), "## Rows"), re.compile(r"^\d+$"))
|
||||
ledger = parse_table(section(doc.splitlines(), "## Source ledger"), LEDGER_ID)
|
||||
by_slug = {c[1]: c for _, c in ledger}
|
||||
phrases = {}
|
||||
for rid, c in rows:
|
||||
if len(c) == 9 and c[8] in ("documented", "contradicted"):
|
||||
q = re.findall(r'"([^"]{15,})"', c[4])
|
||||
phrases[by_slug.get(c[2], [None, None])[1]] = q[0] if q else None
|
||||
|
||||
moved, unreachable, checked = [], [], 0
|
||||
for lid, c in ledger:
|
||||
slug, url, sha = c[1], c[2], c[5]
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "volt-q114/1"})
|
||||
body = urllib.request.urlopen(req, timeout=45).read()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
unreachable.append("%s (%s)" % (slug, type(exc).__name__))
|
||||
continue
|
||||
checked += 1
|
||||
if hashlib.sha256(body).hexdigest() != sha:
|
||||
moved.append(slug)
|
||||
want = phrases.get(slug)
|
||||
if want:
|
||||
text = re.sub(r"\s+", " ", body.decode("utf-8", "replace"))
|
||||
if re.sub(r"\s+", " ", want) not in text:
|
||||
moved.append(slug + "(quote)")
|
||||
if verbose:
|
||||
print(" live: checked %d/%d url(s); unreachable %s"
|
||||
% (checked, len(ledger), unreachable or "none"))
|
||||
if checked == 0:
|
||||
return "FAULT", "no URL could be re-read -- a live leg that fetched nothing is not a pass"
|
||||
if moved:
|
||||
return "OK", "%d/%d re-read, drifted: %s" % (checked, len(ledger), ", ".join(moved[:6]))
|
||||
return "OK", "%d/%d re-read, all digests and quotes unchanged" % (checked, len(ledger))
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--repo", default=DEFAULT_REPO)
|
||||
ap.add_argument("--live", action="store_true")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
repo = os.path.abspath(a.repo)
|
||||
if not os.path.isdir(repo):
|
||||
print("q114: could not grade -- no such directory: " + repo)
|
||||
return 2
|
||||
if not os.path.isfile(os.path.join(repo, DOC)):
|
||||
print("q114: could not grade -- no %s under %s" % (DOC, repo))
|
||||
return 2
|
||||
|
||||
legs = [("document", leg_document), ("mutants", leg_mutants)]
|
||||
if a.live:
|
||||
legs.append(("live", lambda r: leg_live(r, a.verbose)))
|
||||
faults = 0
|
||||
print("q114: %s" % repo)
|
||||
for name, fn in legs:
|
||||
verdict, detail = fn(repo)
|
||||
if verdict != "OK":
|
||||
faults += 1
|
||||
print(" %-9s %-5s %s" % (name, verdict, detail))
|
||||
print("q114: %d leg(s), %d passed, %d fault(s)"
|
||||
% (len(legs), len(legs) - faults, faults))
|
||||
return 1 if faults else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user