#!/usr/bin/env python3
"""Build the unified stackchain magazine catalog from the three source JSONs
(raw transcriptions, batch2 retries, batch3 retries) into a table (JSON + MD)."""
import json
import os
import re

RAW = '/root/stackchain_catalog_raw.json'
BATCH2 = '/root/stackchain_catalog_batch2.json'
BATCH3 = '/root/stackchain_catalog_batch3.json'
OUT_JSON = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.json'
OUT_MD = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.md'
ERROR_RE = re.compile(r'ERROR: (.*?)(?:, full=|$)')
TYPE_RE = re.compile(r'"type"\s*:\s*"(.*?)"')
TEXT_RE = re.compile(r'"text"\s*:\s*"([\s\S]*?)"')


def load(p):
    with open(p, encoding='utf-8') as f:
        return json.load(f)


def clean_text(v):
    """Turn literal backslash-n sequences held inside the source into real
    newlines so the table's Detail section lines wrap cleanly."""
    return v.replace('\\n', '\n').replace('\\\\n', '\n')


def parse_entry_value(v):
    """Return a {'type','text'} dict for one source entry.

    Strict JSON first; if the value is truncated mid-object (the raw
    img_67eedaf526e7.jpg transcription is cut off mid-'identifiers' string),
    fall back to regex extraction of the 'type'/'text' substrings, which are
    otherwise intact.
    """
    if not isinstance(v, str) or not v.strip():
        return None
    try:
        d = json.loads(v)
        if isinstance(d, dict):
            return d
    except Exception:
        pass
    m = TYPE_RE.search(v)
    text_m = TEXT_RE.search(v)
    return {
        'type': m.group(1) if m else '(unknown)',
        'text': clean_text(text_m.group(1)) if text_m else '',
    }


def main():
    raw = load(RAW)
    b2 = load(BATCH2)
    b3 = load(BATCH3)

    entries = []
    # --- raw transcriptions ---
    for idx, (fname, v) in enumerate(raw.items(), 1):
        d = parse_entry_value(v)
        if d and d.get('text'):
            entries.append({
                'no': idx,
                'filename': fname,
                'source': 'raw',
                'status': 'transcribed',
                'type': d.get('type') or '(unknown)',
                'text': d['text'],
                'words': len(d['text'].split()),
            })
        else:
            entries.append({
                'no': idx,
                'filename': fname,
                'source': 'raw',
                'status': 'empty',
                'type': '(not transcribed)',
                'text': '',
                'words': 0,
                'error': 'empty transcription in raw source',
            })

    # --- batch2 retries (retries of images the raw transcription left empty) ---
    for idx, (fname, v) in enumerate(b2.items(), 1):
        d = parse_entry_value(v)
        if d and d.get('text'):
            entries.append({
                'no': idx, 'filename': fname, 'source': 'batch2',
                'status': 'transcribed',
                'type': d.get('type') or '(unknown)',
                'text': clean_text(d.get('text') or ''),
                'visual': '', 'identifiers': '',
                'words': len((d.get('text') or '').split()),
            })
        else:
            m = ERROR_RE.search(v or '')
            err = m.group(1) if m else v[:80]
            entries.append({
                'no': idx, 'filename': fname, 'source': 'batch2',
                'status': 'error',
                'type': '(retry failed)',
                'text': '', 'visual': '', 'identifiers': '',
                'words': 0,
                'error': err,
            })

    # --- batch3 transcriptions ---
    for idx, (fname, v) in enumerate(b3.items(), 1):
        d = parse_entry_value(v)
        if d and d.get('text'):
            entries.append({
                'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
                'status': 'transcribed',
                'type': d.get('type') or '(unknown)',
                'text': clean_text(d.get('text') or ''),
                'visual': '', 'identifiers': '',
                'words': len((d.get('text') or '').split()),
            })
        else:
            entries.append({
                'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
                'status': 'error',
                'type': '(retry failed)',
                'text': '', 'visual': '', 'identifiers': '',
                'words': 0,
                'error': 'empty transcription in batch3 source',
            })

    # Order by filename so a retry is adjacent to the entry it re-ran.
    entries.sort(key=lambda e: e['filename'])

    for e in entries:
        e['no'] = None  # renumber after sort
    for i, e in enumerate(entries, 1):
        e['no'] = i

    with open(OUT_JSON, 'w', encoding='utf-8') as f:
        json.dump(entries, f, indent=2, ensure_ascii=False)

    # Markdown table
    lines = []
    lines.append('# Stackchain Magazine — catalog (extracted from source JSONs)')
    lines.append('')
    lines.append('Source: `stackchain_catalog_raw.json` (raw transcriptions), '
                 '`stackchain_catalog_batch2.json` (retries), '
                 '`stackchain_catalog_batch3.json` (retries). '
                 'One row per catalog image; `raw` entries with no transcription '
                 'are reproduced as `(empty)` and their retries in batch2/batch3 are adjacent.')
    lines.append('')
    lines.append('| No | File | Source | Type | Status | Words |')
    lines.append('|---:|---|----|----|----|---:|')
    for e in entries:
        lines.append(f"| {e['no']} | `{e['filename']}` | {e['source']} | {e['type']} | {e['status']} | {e['words']} |")
    lines.append('')
    lines.append('## Details')
    for e in entries:
        if e.get('text'):
            lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
            lines.append('')
            lines.append('Type: ' + str(e['type']))
            lines.append('')
            lines.append('Text:')
            lines.append('')
            for ln in e['text'].split('\n'):
                lines.append('> ' + ln)
            lines.append('')
        elif e.get('error'):
            lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
            lines.append('')
            lines.append('Error: ' + e['error'])
            lines.append('')
    text = '\n'.join(lines)
    with open(OUT_MD, 'w', encoding='utf-8') as f:
        f.write(text)

    # Summary
    from collections import Counter
    c = Counter(e['status'] for e in entries)
    print('wrote', OUT_JSON, 'and', OUT_MD)
    print('total entries:', len(entries))
    print('by status:', dict(c))
    print('by source:', {s: sum(1 for e in entries if e['source'] == s) for s in sorted(set(e['source'] for e in entries))})

if __name__ == '__main__':
    main()