333 lines
12 KiB
Python
333 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""figure-label-audit/audit.py - independent re-derivation of the vision-label split.
|
|
|
|
Question this tool answers
|
|
--------------------------
|
|
`transformer-archive/manifest.json` claims a 13-item "vision pass" over
|
|
`figure1.png`, split into 3 labels that also occur verbatim in the paper text
|
|
("present") and 10 that do not ("figure-only").
|
|
|
|
`transformer-archive/verify.py` only proves the SPLIT is internally consistent:
|
|
it re-runs `label in source` on the same labels the manifest already holds. A
|
|
label that was never in the figure at all would pass that check unchanged. This
|
|
tool closes the other half of the loop by reading the labels off the pixels with
|
|
tesseract - software independent of the vision model that produced the original
|
|
caption - and grading every manifest label against that reading.
|
|
|
|
Grouping model
|
|
--------------
|
|
Tesseract boxes are in PNG pixel coordinates (this figure is 1520x2239). Two
|
|
steps turn the word list into candidate label strings:
|
|
|
|
1. LINE CLUSTERS - words whose vertical extents overlap by at least 50% of the
|
|
smaller word's height are on the same text line. Within a cluster, words are
|
|
grouped by horizontal gaps, but the gap threshold is scaled to the words on
|
|
that line (max(60px, 1.2x the cluster's mean word width)), so a wide label
|
|
like "Positional Encoding" is not split apart by the space between its two
|
|
words. This matters: a fixed threshold either splits wide labels or joins
|
|
words across a gap that is not a word boundary.
|
|
|
|
2. UNION GROUPS - line clusters that share a word are merged into one group.
|
|
The figure prints some labels as two rows that tesseract returns as
|
|
overlapping boxes ("Output" and "Probabilities" come back on rows 6px
|
|
apart), so those two rows are one label on the page, not two. Union-find
|
|
over shared words recovers them without guessing a vertical distance.
|
|
|
|
Why the group is the unit, not a distance threshold
|
|
---------------------------------------------------
|
|
An earlier revision linked words by physical CONTAINMENT (the upper word's
|
|
extent inside the lower word's). That is wrong for this figure: the connector
|
|
between its two columns is a horizontal rule that is *wider* than every label,
|
|
so it swallows the whole column and merges labels that are not one label. The
|
|
defect was measured, not assumed - with containment the tool reported
|
|
"Output / Probabilities" as one group and "Masked / Multi-Head / Attention" as
|
|
another only by accident of box widths. Union over shared words has no such
|
|
failure: a word belongs to exactly one group, and groups are exactly the
|
|
connected components of the printed label text.
|
|
|
|
Grading per manifest label (tokens = lowercase alphanumerics, stopwords dropped)
|
|
OCR_EXACT >= 0.6 token overlap with one group
|
|
OCR_BOUNDARY at least one token appears in the OCR, but not together in a group
|
|
OCR_ABSENT no token appears in the OCR output at all
|
|
|
|
`present_in_text` is recorded for each label but is NOT part of the verdict -
|
|
that half is `verify.py`'s job, and repeating it here would make this tool a
|
|
restatement rather than an independent check.
|
|
|
|
Exit contract
|
|
0 clean - every manifest label graded OCR_EXACT
|
|
1 a graded fault (a manifest label not read off the pixels)
|
|
2 could not grade (missing file, unparseable manifest, or no OCR signal)
|
|
|
|
Usage: audit.py [--png PATH] [--manifest PATH] [--report PATH]
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
# Tokens with no identifying content, dropped before scoring. Single letters are
|
|
# dropped as a class: this diagram's arrows and connector are read by tesseract
|
|
# as one-letter glyphs ('|', 'J', 'l'), so a single letter is not evidence of a
|
|
# label either way.
|
|
STOPWORDS = frozenset({"and", "the", "a", "an", "of", "or", "to", "in", "on"})
|
|
|
|
# Minimum token overlap (Jaccard) for a group to be counted as its label.
|
|
OVERLAP = 0.6
|
|
|
|
# Vertical overlap, as a fraction of the smaller word's height, for two words to
|
|
# be on the same line.
|
|
LINE_OVERLAP = 0.5
|
|
|
|
# Fixed part of the horizontal gap allowed between two words on the same line.
|
|
GAP_FLOOR = 60
|
|
|
|
# Multiplier on a line cluster's mean word width for the same gap.
|
|
GAP_SCALE = 1.2
|
|
|
|
|
|
def tokenize(text):
|
|
"""Normalise to comparable tokens: lowercase, alphanumerics, stopwords dropped."""
|
|
out = []
|
|
for part in re.split(r"[^A-Za-z0-9]+", text.lower()):
|
|
if part and part not in STOPWORDS and len(part) > 1:
|
|
out.append(part)
|
|
return out
|
|
|
|
|
|
def read_ocr(png_path):
|
|
"""Run tesseract over the PNG; return (words, meta).
|
|
|
|
words - list of dicts {x, y, w, h, raw} in PNG pixel coordinates.
|
|
Raises RuntimeError when tesseract is unavailable.
|
|
"""
|
|
exe = shutil.which("tesseract")
|
|
if not exe:
|
|
raise RuntimeError("tesseract not found on PATH")
|
|
argv = [exe, png_path, "stdout", "--psm", "11", "tsv"]
|
|
proc = subprocess.run(argv, capture_output=True, text=True, timeout=300)
|
|
words = []
|
|
for line in proc.stdout.splitlines()[1:]:
|
|
cols = line.split("\t")
|
|
if len(cols) < 12:
|
|
continue
|
|
raw = cols[11].strip()
|
|
if not raw:
|
|
continue
|
|
try:
|
|
x, y, w, h = (int(cols[6]), int(cols[7]), int(cols[8]), int(cols[9]))
|
|
except ValueError:
|
|
continue
|
|
words.append({"x": x, "y": y, "w": w, "h": h, "raw": raw})
|
|
meta = {"argv": argv, "rc": proc.returncode, "stderr": proc.stderr.strip()}
|
|
return words, meta
|
|
|
|
|
|
def join_hyphen_splits(words):
|
|
"""Rejoin a word ending in '-' with the word after it on the same line."""
|
|
out = []
|
|
for w in sorted(words, key=lambda t: (t["y"], t["x"])):
|
|
if out and out[-1]["raw"].endswith("-"):
|
|
prev = out[-1]
|
|
prev["raw"] = prev["raw"] + w["raw"]
|
|
prev["w"] = (w["x"] + w["w"]) - prev["x"]
|
|
prev["h"] = max(prev["h"], w["h"])
|
|
prev["merged"] = True
|
|
else:
|
|
out.append(dict(w))
|
|
return out
|
|
|
|
|
|
def same_line(a, b):
|
|
"""True when two OCR words sit on one text line."""
|
|
top = max(a["y"], b["y"])
|
|
bottom = min(a["y"] + a["h"], b["y"] + b["h"])
|
|
overlap = max(0, bottom - top)
|
|
return overlap >= LINE_OVERLAP * min(a["h"], b["h"])
|
|
|
|
|
|
def line_clusters(words):
|
|
"""Group OCR words into text lines: sorted by (y, x), split at vertical gaps."""
|
|
clusters = []
|
|
for w in sorted(words, key=lambda t: (t["y"], t["x"])):
|
|
if clusters and same_line(clusters[-1][-1], w):
|
|
clusters[-1].append(w)
|
|
else:
|
|
clusters.append([w])
|
|
return clusters
|
|
|
|
|
|
def line_groups(cluster):
|
|
"""Split one line cluster into runs of words separated by horizontal gaps."""
|
|
if not cluster:
|
|
return []
|
|
mean_w = sum(w["w"] for w in cluster) / float(len(cluster))
|
|
gap = max(GAP_FLOOR, GAP_SCALE * mean_w)
|
|
out = [[cluster[0]]]
|
|
for w in sorted(cluster, key=lambda t: t["x"]):
|
|
if w is cluster[0]:
|
|
continue
|
|
if w["x"] - (out[-1][-1]["x"] + out[-1][-1]["w"]) <= gap:
|
|
out[-1].append(w)
|
|
else:
|
|
out.append([w])
|
|
return out
|
|
|
|
|
|
def groups_from_words(words):
|
|
"""All candidate label groups: line groups, merged where they share a word."""
|
|
parent = list(range(len(words)))
|
|
index = {id(w): i for i, w in enumerate(words)}
|
|
|
|
def find(i):
|
|
while parent[i] != i:
|
|
parent[i] = parent[parent[i]]
|
|
i = parent[i]
|
|
return i
|
|
|
|
def union(i, j):
|
|
ri, rj = find(i), find(j)
|
|
if ri != rj:
|
|
parent[rj] = ri
|
|
|
|
runs = []
|
|
for cluster in line_clusters(words):
|
|
for run in line_groups(cluster):
|
|
if len(run) == 1:
|
|
runs.append(run)
|
|
else:
|
|
runs.append(run)
|
|
for a, b in zip(run, run[1:]):
|
|
union(index[id(a)], index[id(b)])
|
|
buckets = {}
|
|
for w in words:
|
|
buckets.setdefault(find(index[id(w)]), []).append(w)
|
|
return [sorted(v, key=lambda t: (t["y"], t["x"])) for v in buckets.values()]
|
|
|
|
|
|
def overlap(a, b):
|
|
sa, sb = set(a), set(b)
|
|
return len(sa & sb) / float(len(sa | sb)) if (sa | sb) else 0.0
|
|
|
|
|
|
def score_label(label, groups):
|
|
"""Return (status, matched_words) for one manifest label."""
|
|
toks = tokenize(label)
|
|
if not toks:
|
|
return "OCR_ABSENT", []
|
|
scored = []
|
|
for g in groups:
|
|
gt = tokenize(" ".join(w["raw"] for w in g))
|
|
if gt:
|
|
scored.append((overlap(toks, gt), g))
|
|
scored.sort(key=lambda t: -t[0])
|
|
if scored and scored[0][0] >= OVERLAP:
|
|
return "OCR_EXACT", scored[0][1]
|
|
all_tokens = set()
|
|
for _, g in scored:
|
|
all_tokens.update(tokenize(" ".join(w["raw"] for w in g)))
|
|
if any(t in all_tokens for t in toks):
|
|
return "OCR_BOUNDARY", []
|
|
return "OCR_ABSENT", []
|
|
|
|
|
|
def main():
|
|
here = os.path.dirname(os.path.abspath(__file__))
|
|
default_dir = os.path.normpath(os.path.join(here, "..", "transformer-archive"))
|
|
ap = argparse.ArgumentParser(description="re-derive figure labels from pixels")
|
|
ap.add_argument("--png", default=os.path.join(default_dir, "figure1.png"))
|
|
ap.add_argument("--manifest", default=os.path.join(default_dir, "manifest.json"))
|
|
ap.add_argument("--report", default=os.path.join(here, "report.json"))
|
|
a = ap.parse_args()
|
|
|
|
for p, what in ((a.png, "PNG"), (a.manifest, "manifest")):
|
|
if not os.path.isfile(p):
|
|
print("UNGRADABLE - no %s at %s" % (what, p))
|
|
return 2
|
|
try:
|
|
man = json.load(open(a.manifest, encoding="utf-8"))
|
|
except Exception as e:
|
|
print("UNGRADABLE - manifest does not parse: %r" % e)
|
|
return 2
|
|
labels = man.get("vision_labels")
|
|
if not labels or not isinstance(labels, list):
|
|
print("UNGRADABLE - manifest has no 'vision_labels' list")
|
|
return 2
|
|
source_path = os.path.join(os.path.dirname(a.manifest),
|
|
man.get("source", {}).get("snapshot", "SOURCE-text.md"))
|
|
if not os.path.isfile(source_path):
|
|
print("UNGRADABLE - source snapshot missing: %s" % source_path)
|
|
return 2
|
|
source = open(source_path, encoding="utf-8").read()
|
|
|
|
try:
|
|
raw_words, meta = read_ocr(a.png)
|
|
except Exception as e:
|
|
print("UNGRADABLE - OCR failed: %r" % e)
|
|
return 2
|
|
if not raw_words:
|
|
print("UNGRADABLE - OCR returned no words (rc=%s) %s"
|
|
% (meta.get("rc"), (meta.get("stderr") or "")[:200]))
|
|
return 2
|
|
|
|
words = join_hyphen_splits(raw_words)
|
|
groups = groups_from_words(words)
|
|
|
|
rows = []
|
|
faults = []
|
|
for lab in labels:
|
|
status, run = score_label(lab, groups)
|
|
rows.append({
|
|
"label": lab,
|
|
"ocr_status": status,
|
|
"present_in_text": lab in source,
|
|
"tokens": tokenize(lab),
|
|
"ocr_run": [{"text": w["raw"], "x": w["x"], "y": w["y"]} for w in run],
|
|
})
|
|
if status != "OCR_EXACT":
|
|
faults.append((lab, status))
|
|
|
|
report = {
|
|
"artifact": "figure-label-audit",
|
|
"curator": "sphinx",
|
|
"png": os.path.abspath(a.png),
|
|
"png_bytes": os.path.getsize(a.png),
|
|
"manifest": os.path.abspath(a.manifest),
|
|
"source_snapshot": os.path.abspath(source_path),
|
|
"ocr_engine": meta["argv"][0],
|
|
"ocr_psm": "11",
|
|
"ocr_word_count_raw": len(raw_words),
|
|
"ocr_word_count_joined": len(words),
|
|
"ocr_group_count": len(groups),
|
|
"labels_graded": len(rows),
|
|
"ocr_exact": sum(1 for r in rows if r["ocr_status"] == "OCR_EXACT"),
|
|
"ocr_boundary": sum(1 for r in rows if r["ocr_status"] == "OCR_BOUNDARY"),
|
|
"ocr_absent": sum(1 for r in rows if r["ocr_status"] == "OCR_ABSENT"),
|
|
"faults": [{"label": l, "ocr_status": s} for l, s in faults],
|
|
"rows": rows,
|
|
"ocr_groups": [
|
|
[{"text": w["raw"], "x": w["x"], "y": w["y"]} for w in g] for g in groups
|
|
],
|
|
}
|
|
with open(a.report, "w", encoding="utf-8") as f:
|
|
json.dump(report, f, indent=2, ensure_ascii=False)
|
|
f.write("\n")
|
|
|
|
print("figure-label-audit: %d labels graded against %d OCR words "
|
|
"(%d groups)" % (len(rows), len(words), len(groups)))
|
|
print(" OCR_EXACT %d" % report["ocr_exact"])
|
|
print(" OCR_BOUNDARY %d" % report["ocr_boundary"])
|
|
print(" OCR_ABSENT %d" % report["ocr_absent"])
|
|
for lab, status in faults:
|
|
print(" FAULT %s - %s" % (lab, status))
|
|
print("report: %s" % os.path.abspath(a.report))
|
|
|
|
return 1 if faults else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|