187 lines
6.7 KiB
Python
187 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the unified stackchain magazine catalog from the three source JSONs
|
|
(raw transcriptions, batch2 retries, batch3 retries) into a table (JSON + MD)."""
|
|
import json
|
|
import os
|
|
import re
|
|
|
|
RAW = '/root/stackchain_catalog_raw.json'
|
|
BATCH2 = '/root/stackchain_catalog_batch2.json'
|
|
BATCH3 = '/root/stackchain_catalog_batch3.json'
|
|
OUT_JSON = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.json'
|
|
OUT_MD = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.md'
|
|
ERROR_RE = re.compile(r'ERROR: (.*?)(?:, full=|$)')
|
|
TYPE_RE = re.compile(r'"type"\s*:\s*"(.*?)"')
|
|
TEXT_RE = re.compile(r'"text"\s*:\s*"([\s\S]*?)"')
|
|
|
|
|
|
def load(p):
|
|
with open(p, encoding='utf-8') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def clean_text(v):
|
|
"""Turn literal backslash-n sequences held inside the source into real
|
|
newlines so the table's Detail section lines wrap cleanly."""
|
|
return v.replace('\\n', '\n').replace('\\\\n', '\n')
|
|
|
|
|
|
def parse_entry_value(v):
|
|
"""Return a {'type','text'} dict for one source entry.
|
|
|
|
Strict JSON first; if the value is truncated mid-object (the raw
|
|
img_67eedaf526e7.jpg transcription is cut off mid-'identifiers' string),
|
|
fall back to regex extraction of the 'type'/'text' substrings, which are
|
|
otherwise intact.
|
|
"""
|
|
if not isinstance(v, str) or not v.strip():
|
|
return None
|
|
try:
|
|
d = json.loads(v)
|
|
if isinstance(d, dict):
|
|
return d
|
|
except Exception:
|
|
pass
|
|
m = TYPE_RE.search(v)
|
|
text_m = TEXT_RE.search(v)
|
|
return {
|
|
'type': m.group(1) if m else '(unknown)',
|
|
'text': clean_text(text_m.group(1)) if text_m else '',
|
|
}
|
|
|
|
|
|
def main():
|
|
raw = load(RAW)
|
|
b2 = load(BATCH2)
|
|
b3 = load(BATCH3)
|
|
|
|
entries = []
|
|
# --- raw transcriptions ---
|
|
for idx, (fname, v) in enumerate(raw.items(), 1):
|
|
d = parse_entry_value(v)
|
|
if d and d.get('text'):
|
|
entries.append({
|
|
'no': idx,
|
|
'filename': fname,
|
|
'source': 'raw',
|
|
'status': 'transcribed',
|
|
'type': d.get('type') or '(unknown)',
|
|
'text': d['text'],
|
|
'words': len(d['text'].split()),
|
|
})
|
|
else:
|
|
entries.append({
|
|
'no': idx,
|
|
'filename': fname,
|
|
'source': 'raw',
|
|
'status': 'empty',
|
|
'type': '(not transcribed)',
|
|
'text': '',
|
|
'words': 0,
|
|
'error': 'empty transcription in raw source',
|
|
})
|
|
|
|
# --- batch2 retries (retries of images the raw transcription left empty) ---
|
|
for idx, (fname, v) in enumerate(b2.items(), 1):
|
|
d = parse_entry_value(v)
|
|
if d and d.get('text'):
|
|
entries.append({
|
|
'no': idx, 'filename': fname, 'source': 'batch2',
|
|
'status': 'transcribed',
|
|
'type': d.get('type') or '(unknown)',
|
|
'text': clean_text(d.get('text') or ''),
|
|
'visual': '', 'identifiers': '',
|
|
'words': len((d.get('text') or '').split()),
|
|
})
|
|
else:
|
|
m = ERROR_RE.search(v or '')
|
|
err = m.group(1) if m else v[:80]
|
|
entries.append({
|
|
'no': idx, 'filename': fname, 'source': 'batch2',
|
|
'status': 'error',
|
|
'type': '(retry failed)',
|
|
'text': '', 'visual': '', 'identifiers': '',
|
|
'words': 0,
|
|
'error': err,
|
|
})
|
|
|
|
# --- batch3 transcriptions ---
|
|
for idx, (fname, v) in enumerate(b3.items(), 1):
|
|
d = parse_entry_value(v)
|
|
if d and d.get('text'):
|
|
entries.append({
|
|
'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
|
|
'status': 'transcribed',
|
|
'type': d.get('type') or '(unknown)',
|
|
'text': clean_text(d.get('text') or ''),
|
|
'visual': '', 'identifiers': '',
|
|
'words': len((d.get('text') or '').split()),
|
|
})
|
|
else:
|
|
entries.append({
|
|
'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
|
|
'status': 'error',
|
|
'type': '(retry failed)',
|
|
'text': '', 'visual': '', 'identifiers': '',
|
|
'words': 0,
|
|
'error': 'empty transcription in batch3 source',
|
|
})
|
|
|
|
# Order by filename so a retry is adjacent to the entry it re-ran.
|
|
entries.sort(key=lambda e: e['filename'])
|
|
|
|
for e in entries:
|
|
e['no'] = None # renumber after sort
|
|
for i, e in enumerate(entries, 1):
|
|
e['no'] = i
|
|
|
|
with open(OUT_JSON, 'w', encoding='utf-8') as f:
|
|
json.dump(entries, f, indent=2, ensure_ascii=False)
|
|
|
|
# Markdown table
|
|
lines = []
|
|
lines.append('# Stackchain Magazine — catalog (extracted from source JSONs)')
|
|
lines.append('')
|
|
lines.append('Source: `stackchain_catalog_raw.json` (raw transcriptions), '
|
|
'`stackchain_catalog_batch2.json` (retries), '
|
|
'`stackchain_catalog_batch3.json` (retries). '
|
|
'One row per catalog image; `raw` entries with no transcription '
|
|
'are reproduced as `(empty)` and their retries in batch2/batch3 are adjacent.')
|
|
lines.append('')
|
|
lines.append('| No | File | Source | Type | Status | Words |')
|
|
lines.append('|---:|---|----|----|----|---:|')
|
|
for e in entries:
|
|
lines.append(f"| {e['no']} | `{e['filename']}` | {e['source']} | {e['type']} | {e['status']} | {e['words']} |")
|
|
lines.append('')
|
|
lines.append('## Details')
|
|
for e in entries:
|
|
if e.get('text'):
|
|
lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
|
|
lines.append('')
|
|
lines.append('Type: ' + str(e['type']))
|
|
lines.append('')
|
|
lines.append('Text:')
|
|
lines.append('')
|
|
for ln in e['text'].split('\n'):
|
|
lines.append('> ' + ln)
|
|
lines.append('')
|
|
elif e.get('error'):
|
|
lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
|
|
lines.append('')
|
|
lines.append('Error: ' + e['error'])
|
|
lines.append('')
|
|
text = '\n'.join(lines)
|
|
with open(OUT_MD, 'w', encoding='utf-8') as f:
|
|
f.write(text)
|
|
|
|
# Summary
|
|
from collections import Counter
|
|
c = Counter(e['status'] for e in entries)
|
|
print('wrote', OUT_JSON, 'and', OUT_MD)
|
|
print('total entries:', len(entries))
|
|
print('by status:', dict(c))
|
|
print('by source:', {s: sum(1 for e in entries if e['source'] == s) for s in sorted(set(e['source'] for e in entries))})
|
|
|
|
if __name__ == '__main__':
|
|
main()
|