ledger: stackchain magazine catalog extraction (18-row table + CHECK)
This commit is contained in:
1 parent
b21ad1b324
commit
155c21e153
4 files changed
+797
-54
No files matched your search
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the unified stackchain magazine catalog from the three source JSONs
|
||||
(raw transcriptions, batch2 retries, batch3 retries) into a table (JSON + MD)."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
RAW = '/root/stackchain_catalog_raw.json'
|
||||
BATCH2 = '/root/stackchain_catalog_batch2.json'
|
||||
BATCH3 = '/root/stackchain_catalog_batch3.json'
|
||||
OUT_JSON = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.json'
|
||||
OUT_MD = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.md'
|
||||
ERROR_RE = re.compile(r'ERROR: (.*?)(?:, full=|$)')
|
||||
TYPE_RE = re.compile(r'"type"\s*:\s*"(.*?)"')
|
||||
TEXT_RE = re.compile(r'"text"\s*:\s*"([\s\S]*?)"')
|
||||
|
||||
|
||||
def load(p):
|
||||
with open(p, encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def clean_text(v):
|
||||
"""Turn literal backslash-n sequences held inside the source into real
|
||||
newlines so the table's Detail section lines wrap cleanly."""
|
||||
return v.replace('\\n', '\n').replace('\\\\n', '\n')
|
||||
|
||||
|
||||
def parse_entry_value(v):
|
||||
"""Return a {'type','text'} dict for one source entry.
|
||||
|
||||
Strict JSON first; if the value is truncated mid-object (the raw
|
||||
img_67eedaf526e7.jpg transcription is cut off mid-'identifiers' string),
|
||||
fall back to regex extraction of the 'type'/'text' substrings, which are
|
||||
otherwise intact.
|
||||
"""
|
||||
if not isinstance(v, str) or not v.strip():
|
||||
return None
|
||||
try:
|
||||
d = json.loads(v)
|
||||
if isinstance(d, dict):
|
||||
return d
|
||||
except Exception:
|
||||
pass
|
||||
m = TYPE_RE.search(v)
|
||||
text_m = TEXT_RE.search(v)
|
||||
return {
|
||||
'type': m.group(1) if m else '(unknown)',
|
||||
'text': clean_text(text_m.group(1)) if text_m else '',
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
raw = load(RAW)
|
||||
b2 = load(BATCH2)
|
||||
b3 = load(BATCH3)
|
||||
|
||||
entries = []
|
||||
# --- raw transcriptions ---
|
||||
for idx, (fname, v) in enumerate(raw.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({
|
||||
'no': idx,
|
||||
'filename': fname,
|
||||
'source': 'raw',
|
||||
'status': 'transcribed',
|
||||
'type': d.get('type') or '(unknown)',
|
||||
'text': d['text'],
|
||||
'words': len(d['text'].split()),
|
||||
})
|
||||
else:
|
||||
entries.append({
|
||||
'no': idx,
|
||||
'filename': fname,
|
||||
'source': 'raw',
|
||||
'status': 'empty',
|
||||
'type': '(not transcribed)',
|
||||
'text': '',
|
||||
'words': 0,
|
||||
'error': 'empty transcription in raw source',
|
||||
})
|
||||
|
||||
# --- batch2 retries (retries of images the raw transcription left empty) ---
|
||||
for idx, (fname, v) in enumerate(b2.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({
|
||||
'no': idx, 'filename': fname, 'source': 'batch2',
|
||||
'status': 'transcribed',
|
||||
'type': d.get('type') or '(unknown)',
|
||||
'text': clean_text(d.get('text') or ''),
|
||||
'visual': '', 'identifiers': '',
|
||||
'words': len((d.get('text') or '').split()),
|
||||
})
|
||||
else:
|
||||
m = ERROR_RE.search(v or '')
|
||||
err = m.group(1) if m else v[:80]
|
||||
entries.append({
|
||||
'no': idx, 'filename': fname, 'source': 'batch2',
|
||||
'status': 'error',
|
||||
'type': '(retry failed)',
|
||||
'text': '', 'visual': '', 'identifiers': '',
|
||||
'words': 0,
|
||||
'error': err,
|
||||
})
|
||||
|
||||
# --- batch3 transcriptions ---
|
||||
for idx, (fname, v) in enumerate(b3.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({
|
||||
'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
|
||||
'status': 'transcribed',
|
||||
'type': d.get('type') or '(unknown)',
|
||||
'text': clean_text(d.get('text') or ''),
|
||||
'visual': '', 'identifiers': '',
|
||||
'words': len((d.get('text') or '').split()),
|
||||
})
|
||||
else:
|
||||
entries.append({
|
||||
'no': idx + len(raw) + len(b2), 'filename': fname, 'source': 'batch3',
|
||||
'status': 'error',
|
||||
'type': '(retry failed)',
|
||||
'text': '', 'visual': '', 'identifiers': '',
|
||||
'words': 0,
|
||||
'error': 'empty transcription in batch3 source',
|
||||
})
|
||||
|
||||
# Order by filename so a retry is adjacent to the entry it re-ran.
|
||||
entries.sort(key=lambda e: e['filename'])
|
||||
|
||||
for e in entries:
|
||||
e['no'] = None # renumber after sort
|
||||
for i, e in enumerate(entries, 1):
|
||||
e['no'] = i
|
||||
|
||||
with open(OUT_JSON, 'w', encoding='utf-8') as f:
|
||||
json.dump(entries, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Markdown table
|
||||
lines = []
|
||||
lines.append('# Stackchain Magazine — catalog (extracted from source JSONs)')
|
||||
lines.append('')
|
||||
lines.append('Source: `stackchain_catalog_raw.json` (raw transcriptions), '
|
||||
'`stackchain_catalog_batch2.json` (retries), '
|
||||
'`stackchain_catalog_batch3.json` (retries). '
|
||||
'One row per catalog image; `raw` entries with no transcription '
|
||||
'are reproduced as `(empty)` and their retries in batch2/batch3 are adjacent.')
|
||||
lines.append('')
|
||||
lines.append('| No | File | Source | Type | Status | Words |')
|
||||
lines.append('|---:|---|----|----|----|---:|')
|
||||
for e in entries:
|
||||
lines.append(f"| {e['no']} | `{e['filename']}` | {e['source']} | {e['type']} | {e['status']} | {e['words']} |")
|
||||
lines.append('')
|
||||
lines.append('## Details')
|
||||
for e in entries:
|
||||
if e.get('text'):
|
||||
lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
|
||||
lines.append('')
|
||||
lines.append('Type: ' + str(e['type']))
|
||||
lines.append('')
|
||||
lines.append('Text:')
|
||||
lines.append('')
|
||||
for ln in e['text'].split('\n'):
|
||||
lines.append('> ' + ln)
|
||||
lines.append('')
|
||||
elif e.get('error'):
|
||||
lines.append(f"### {e['no']}. `{e['filename']}` ({e['source']}, {e['status']})")
|
||||
lines.append('')
|
||||
lines.append('Error: ' + e['error'])
|
||||
lines.append('')
|
||||
text = '\n'.join(lines)
|
||||
with open(OUT_MD, 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
# Summary
|
||||
from collections import Counter
|
||||
c = Counter(e['status'] for e in entries)
|
||||
print('wrote', OUT_JSON, 'and', OUT_MD)
|
||||
print('total entries:', len(entries))
|
||||
print('by status:', dict(c))
|
||||
print('by source:', {s: sum(1 for e in entries if e['source'] == s) for s in sorted(set(e['source'] for e in entries))})
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,66 +1,194 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"filename": "img_67eedaf526e7.jpg",
|
||||
"section": "COVER",
|
||||
"type": "COVER",
|
||||
"text_preview": "STACKCHAIN",
|
||||
"identifiers": "Mascot logo (cartoon monkey/chimp, orange bandana, cursive \"stackchain\" wordmark). \"18+\" age badge. Mirrored/reversed text elements on cover (\"SELF-CUSTODY//BITCOIN ONLY\", graffiti \"MAGAZINE\"). Bylines/author names: MADEX, ISRELI/ISBELL, DENIZ, SKYLER (no @handles). No block-height notation or sat amounts readable."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"filename": "img_fcb2e6a8f066.jpg",
|
||||
"section": "PHOTO OF PERSON HOLDING MAGAZINE",
|
||||
"type": "PHOTO_OF_PERSON_HOLDING_MAGAZINE",
|
||||
"text_preview": "B/H872.372",
|
||||
"identifiers": "Spine call-number label \"B/H872.372\". Cover logo \"MACHINA\". No @handles, avatars, or sats amounts readable."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"filename": "img_72f52468393f.jpg",
|
||||
"section": "PAGE (\"58K MEMES\")",
|
||||
"type": "PAGE",
|
||||
"text_preview": "STACKCHAIN MAGAZINE",
|
||||
"identifiers": "\"STACKCHAIN MAGAZINE\" logo; \"@chain\"; hashtags #GoPio, #DSB; \"D&B GANG\"; \"LBDC STABLECOIN\"; \"Lokepetyfork\". Amounts: 58K, 58k, 59k, 100K, 100k, $417.00, $580.00. No block heights visible."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"filename": "img_d5ada9cee4e9.jpg",
|
||||
"section": "PAGE (\"GO PIOS\")",
|
||||
"type": "PAGE",
|
||||
"text_preview": "DS #GOPIOS",
|
||||
"identifiers": "None (no @handles, watermarks, or sats amounts readable)."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"filename": "img_7333a3df0aac.jpg",
|
||||
"section": "PAGE (\"MEME MUSEUM\")",
|
||||
"type": "PAGE",
|
||||
"text_preview": "LOKOPETTYFORK",
|
||||
"identifiers": "\"@Lokopettyfork\"; \"LokoPettyFork\"/\"Lokopettyfork\"; \"verizon\" logo; \"MSTR\" ticker; Bitcoin logo; \"₿SB\" reference. Amounts: $70,206.87, $10.00, 3,052.52, 3.01%, 58K, 100K, $100, 100, $100.00. **Number \"8722100\" appears (likely a block height, unconfirmed).**"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"no": 1,
|
||||
"filename": "img_05d51da1fd9d.jpg",
|
||||
"section": "COVER (Captain's Log 4)",
|
||||
"type": "COVER",
|
||||
"text_preview": "STACKCHAIN",
|
||||
"identifiers": "No @handles or amounts readable. Bitcoin (₿) symbols on gold coin + octopus head. Lower-left logo: cartoon monkey with red bandana + \"MAGAZINE\" beneath. No block-height notation readable."
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"no": 2,
|
||||
"filename": "img_05d51da1fd9d.jpg",
|
||||
"source": "batch2",
|
||||
"status": "transcribed",
|
||||
"type": "COVER",
|
||||
"text": "STACKCHAIN\nMAGAZINE\nCAPTAIN'S LOG 4 TO BOLDY STACK\nWHERE NO ONE HAS STACKED\nBEFORE. A NEW ALL TIME HIGH\nINTERVIEW WITH LINA SEICHE (THE LITTLE HODLER)\nBY TSELLY\nTHE OBSTACLE IS THE WAY: FINDING HOPE IN BITCOIN\nBY HUMBLE WARRIOR\nTWO YEARS OF STACKCHAIN\nBY ANTHONY VON DESSAUER",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 48
|
||||
},
|
||||
{
|
||||
"no": 3,
|
||||
"filename": "img_125532ea2e01.jpg",
|
||||
"section": "COVER",
|
||||
"type": "COVER",
|
||||
"text_preview": "B/H: 850,683",
|
||||
"identifiers": "**Block height printed on orange band: \"B/H: 850,683\"** — historical record. Brand \"STACKCHAIN\" in cover text. No @handles or sat amounts readable; the only numeric identifier is block height 850,683."
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"no": 4,
|
||||
"filename": "img_125532ea2e01.jpg",
|
||||
"source": "batch2",
|
||||
"status": "transcribed",
|
||||
"type": "COVER",
|
||||
"text": "B/H: 850,683\nBY NOTHING YOU DESERVE\nTOM SAVES 10 STACKCHAIN\nBY THE TIME THE BUYING [unreadable]\nFOMO IN THE [unreadable] HOPE [unreadable]\nBY STACKING SATS, [unreadable]\nBUY STACKCHAIN AND HODL THE LITTLE [unreadable]\n[unreadable]",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 34
|
||||
},
|
||||
{
|
||||
"no": 5,
|
||||
"filename": "img_1325775751e6.jpg",
|
||||
"section": "PAGE (\"MEME MUSEUM\")",
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"no": 6,
|
||||
"filename": "img_1325775751e6.jpg",
|
||||
"source": "batch2",
|
||||
"status": "error",
|
||||
"type": "(retry failed)",
|
||||
"text": "",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 0,
|
||||
"error": "empty content, finish_reason=length"
|
||||
},
|
||||
{
|
||||
"no": 7,
|
||||
"filename": "img_1325775751e6.jpg",
|
||||
"source": "batch3",
|
||||
"status": "transcribed",
|
||||
"type": "PAGE",
|
||||
"text_preview": "[unreadable]CLASSIC",
|
||||
"identifiers": "No @handles visible. **QR code printed on left page edge.** \"Stackchainers\" branding in party meme. Amounts: 1888 (stacked, per caption), 70K, 85.11 (small partially-legible label). No block heights readable."
|
||||
"text": "[unreadable]CLASSIC\nPERO\nional rapper.\nI was a slave yesterday\nMan how did you become so rich?\nFucking Legend\nGFY\nHon. Yellow\nMEME MUSEUM\n[unreadable]\nYou guys should check out coinbase\nStackchainers\nHow did you afford this house Dad?\nWell it all started when Bananahodl stacked 1888 which, of course, Tao forked and then Ant...\nFucking legend\nGFY\nNO.\n70K\n85.11",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 60
|
||||
},
|
||||
{
|
||||
"no": 8,
|
||||
"filename": "img_67eedaf526e7.jpg",
|
||||
"source": "raw",
|
||||
"status": "transcribed",
|
||||
"type": "COVER",
|
||||
"text": "STACKCHAIN\nMAGAZINE\nROUND 5\nINTERVIEW WITH MADEX\nBY ISBELL\n88K CHRONICLES: HOW ICO CONSPIRED\nTO SUPPRESS THE BITCOIN PRICE\nBY DENIZ\nREASONS TO BUY ALTCOINS\nBY SKYLER\nSELF-CUSTODY//BITCOIN ONLY\nMAGAZINE\n18+\nstackchain\nMAGAZINE\nMADE IN [unreadable]\n[unreadable graffiti tags on the owl artwork]",
|
||||
"words": 43
|
||||
},
|
||||
{
|
||||
"no": 9,
|
||||
"filename": "img_72f52468393f.jpg",
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"no": 10,
|
||||
"filename": "img_72f52468393f.jpg",
|
||||
"source": "batch2",
|
||||
"status": "error",
|
||||
"type": "(retry failed)",
|
||||
"text": "",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 0,
|
||||
"error": "empty content, finish_reason=length"
|
||||
},
|
||||
{
|
||||
"no": 11,
|
||||
"filename": "img_72f52468393f.jpg",
|
||||
"source": "batch3",
|
||||
"status": "transcribed",
|
||||
"type": "PAGE",
|
||||
"text": "STACKCHAIN MAGAZINE\n58K MEMES\n58K\n#GoPio\n$417.00\n58k @chain\nJOIN LBDC STABLECOIN\nCHOOSE FREEDOM D&B GANG TO 100K\nWENT BACK TO REMOVE 58K FROM THE CHAIN\nFACK EVERYTHING\nThe halvening\n58k forever\ndon't sell\nwe're so back\nLFG\npump it!!!\n58k forever\nEXPECTATIONS\n100k\n58k\nbitchcoin reality\nU wake up and see the price at 58k\nImmediately u buy and stack to Lokepetyfork\n1, U save the raccoons\n2, U raise the price to 59k\n#DSB to 100k\n58K MAXI\n$580.00\n58K\nPLEBS\nSTACKCHAIN MAGA",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 85
|
||||
},
|
||||
{
|
||||
"no": 12,
|
||||
"filename": "img_7333a3df0aac.jpg",
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"no": 13,
|
||||
"filename": "img_7333a3df0aac.jpg",
|
||||
"source": "batch2",
|
||||
"status": "error",
|
||||
"type": "(retry failed)",
|
||||
"text": "",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 0,
|
||||
"error": "empty content, finish_reason=length"
|
||||
},
|
||||
{
|
||||
"no": 14,
|
||||
"filename": "img_7333a3df0aac.jpg",
|
||||
"source": "batch3",
|
||||
"status": "transcribed",
|
||||
"type": "PAGE",
|
||||
"text": "LOKOPETTYFORK\nGANG FOREVER\n$10.00\nYOU ARE PART OF\nTHE REBEL ALLIANCE,\nAND A TRAITOR!\nLokoPettyFork\nDon't Stop Believing\nBitcoin\n$70,206.87\n▲ 3,052.52 (3.01%) Past Year\nverizon\nMEME\nMUSEUM\n100\nSAY NO TO 58K CBDC STABLECOIN\nCHOOSE FREEDOM ₿SB GANG TO 100K!\nBUY BITCOIN\nBUY BITCOIN\nBUY BITCOIN\nBUY BITCOIN\n$100.00\n@Lokopettyfork\n8722100\nWHAT IS YOUR KEY TO SUCCESS?\nLOOK, OTHERS BUY MSTR, I JUST STACK TO LOKOPETTYFORK IT'S NOT SOOO COMPLICATED\n100\nLokopettyfork",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 73
|
||||
},
|
||||
{
|
||||
"no": 15,
|
||||
"filename": "img_d5ada9cee4e9.jpg",
|
||||
"source": "raw",
|
||||
"status": "empty",
|
||||
"type": "(not transcribed)",
|
||||
"text": "",
|
||||
"words": 0,
|
||||
"error": "empty transcription in raw source"
|
||||
},
|
||||
{
|
||||
"no": 16,
|
||||
"filename": "img_d5ada9cee4e9.jpg",
|
||||
"source": "batch2",
|
||||
"status": "error",
|
||||
"type": "(retry failed)",
|
||||
"text": "",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 0,
|
||||
"error": "empty content, finish_reason=length"
|
||||
},
|
||||
{
|
||||
"no": 17,
|
||||
"filename": "img_d5ada9cee4e9.jpg",
|
||||
"source": "batch3",
|
||||
"status": "transcribed",
|
||||
"type": "PAGE",
|
||||
"text": "DS #GOPIOS\nWhat I think I look like\nGO Pios\nWhat I actually look like\nSay 'Go Pios'\nLEAVE IT ON.\n#GOPIOS\nCarroll University\nBitcoin Twitter\nMe: GOPIOS\n#GoPios\n#Go Pios\nGFY\n#GoPios\nvechains\nElection\n58kgs\nMe, just vibing to GO PIOS\nGO PIOS\nPIZZA? BLOCK? ALL TIME HIGHS?\nGO PIOS!\nSoul ETF\nWEN\nGM\nGOPIOS\nSTOP BELIEV\nCARROLL\n6",
|
||||
"visual": "",
|
||||
"identifiers": "",
|
||||
"words": 60
|
||||
},
|
||||
{
|
||||
"no": 18,
|
||||
"filename": "img_fcb2e6a8f066.jpg",
|
||||
"source": "raw",
|
||||
"status": "transcribed",
|
||||
"type": "PHOTO_OF_PERSON_HOLDING_MAGAZINE",
|
||||
"text": "B/H872.372\n[unreadable]O CHAIN\n[unreadable]INE\n[unreadable]OUND 5\nMACHINA\nSELF-CUSTODY//BIT[unreadable]",
|
||||
"words": 8
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,269 @@
|
||||
# Stackchain Magazine — catalog (extracted from source JSONs)
|
||||
|
||||
Source: `stackchain_catalog_raw.json` (raw transcriptions), `stackchain_catalog_batch2.json` (retries), `stackchain_catalog_batch3.json` (retries). One row per catalog image; `raw` entries with no transcription are reproduced as `(empty)` and their retries in batch2/batch3 are adjacent.
|
||||
|
||||
| No | File | Source | Type | Status | Words |
|
||||
|---:|---|----|----|----|---:|
|
||||
| 1 | `img_05d51da1fd9d.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 2 | `img_05d51da1fd9d.jpg` | batch2 | COVER | transcribed | 48 |
|
||||
| 3 | `img_125532ea2e01.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 4 | `img_125532ea2e01.jpg` | batch2 | COVER | transcribed | 34 |
|
||||
| 5 | `img_1325775751e6.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 6 | `img_1325775751e6.jpg` | batch2 | (retry failed) | error | 0 |
|
||||
| 7 | `img_1325775751e6.jpg` | batch3 | PAGE | transcribed | 60 |
|
||||
| 8 | `img_67eedaf526e7.jpg` | raw | COVER | transcribed | 43 |
|
||||
| 9 | `img_72f52468393f.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 10 | `img_72f52468393f.jpg` | batch2 | (retry failed) | error | 0 |
|
||||
| 11 | `img_72f52468393f.jpg` | batch3 | PAGE | transcribed | 85 |
|
||||
| 12 | `img_7333a3df0aac.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 13 | `img_7333a3df0aac.jpg` | batch2 | (retry failed) | error | 0 |
|
||||
| 14 | `img_7333a3df0aac.jpg` | batch3 | PAGE | transcribed | 73 |
|
||||
| 15 | `img_d5ada9cee4e9.jpg` | raw | (not transcribed) | empty | 0 |
|
||||
| 16 | `img_d5ada9cee4e9.jpg` | batch2 | (retry failed) | error | 0 |
|
||||
| 17 | `img_d5ada9cee4e9.jpg` | batch3 | PAGE | transcribed | 60 |
|
||||
| 18 | `img_fcb2e6a8f066.jpg` | raw | PHOTO_OF_PERSON_HOLDING_MAGAZINE | transcribed | 8 |
|
||||
|
||||
## Details
|
||||
### 1. `img_05d51da1fd9d.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 2. `img_05d51da1fd9d.jpg` (batch2, transcribed)
|
||||
|
||||
Type: COVER
|
||||
|
||||
Text:
|
||||
|
||||
> STACKCHAIN
|
||||
> MAGAZINE
|
||||
> CAPTAIN'S LOG 4 TO BOLDY STACK
|
||||
> WHERE NO ONE HAS STACKED
|
||||
> BEFORE. A NEW ALL TIME HIGH
|
||||
> INTERVIEW WITH LINA SEICHE (THE LITTLE HODLER)
|
||||
> BY TSELLY
|
||||
> THE OBSTACLE IS THE WAY: FINDING HOPE IN BITCOIN
|
||||
> BY HUMBLE WARRIOR
|
||||
> TWO YEARS OF STACKCHAIN
|
||||
> BY ANTHONY VON DESSAUER
|
||||
|
||||
### 3. `img_125532ea2e01.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 4. `img_125532ea2e01.jpg` (batch2, transcribed)
|
||||
|
||||
Type: COVER
|
||||
|
||||
Text:
|
||||
|
||||
> B/H: 850,683
|
||||
> BY NOTHING YOU DESERVE
|
||||
> TOM SAVES 10 STACKCHAIN
|
||||
> BY THE TIME THE BUYING [unreadable]
|
||||
> FOMO IN THE [unreadable] HOPE [unreadable]
|
||||
> BY STACKING SATS, [unreadable]
|
||||
> BUY STACKCHAIN AND HODL THE LITTLE [unreadable]
|
||||
> [unreadable]
|
||||
|
||||
### 5. `img_1325775751e6.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 6. `img_1325775751e6.jpg` (batch2, error)
|
||||
|
||||
Error: empty content, finish_reason=length
|
||||
|
||||
### 7. `img_1325775751e6.jpg` (batch3, transcribed)
|
||||
|
||||
Type: PAGE
|
||||
|
||||
Text:
|
||||
|
||||
> [unreadable]CLASSIC
|
||||
> PERO
|
||||
> ional rapper.
|
||||
> I was a slave yesterday
|
||||
> Man how did you become so rich?
|
||||
> Fucking Legend
|
||||
> GFY
|
||||
> Hon. Yellow
|
||||
> MEME MUSEUM
|
||||
> [unreadable]
|
||||
> You guys should check out coinbase
|
||||
> Stackchainers
|
||||
> How did you afford this house Dad?
|
||||
> Well it all started when Bananahodl stacked 1888 which, of course, Tao forked and then Ant...
|
||||
> Fucking legend
|
||||
> GFY
|
||||
> NO.
|
||||
> 70K
|
||||
> 85.11
|
||||
|
||||
### 8. `img_67eedaf526e7.jpg` (raw, transcribed)
|
||||
|
||||
Type: COVER
|
||||
|
||||
Text:
|
||||
|
||||
> STACKCHAIN
|
||||
> MAGAZINE
|
||||
> ROUND 5
|
||||
> INTERVIEW WITH MADEX
|
||||
> BY ISBELL
|
||||
> 88K CHRONICLES: HOW ICO CONSPIRED
|
||||
> TO SUPPRESS THE BITCOIN PRICE
|
||||
> BY DENIZ
|
||||
> REASONS TO BUY ALTCOINS
|
||||
> BY SKYLER
|
||||
> SELF-CUSTODY//BITCOIN ONLY
|
||||
> MAGAZINE
|
||||
> 18+
|
||||
> stackchain
|
||||
> MAGAZINE
|
||||
> MADE IN [unreadable]
|
||||
> [unreadable graffiti tags on the owl artwork]
|
||||
|
||||
### 9. `img_72f52468393f.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 10. `img_72f52468393f.jpg` (batch2, error)
|
||||
|
||||
Error: empty content, finish_reason=length
|
||||
|
||||
### 11. `img_72f52468393f.jpg` (batch3, transcribed)
|
||||
|
||||
Type: PAGE
|
||||
|
||||
Text:
|
||||
|
||||
> STACKCHAIN MAGAZINE
|
||||
> 58K MEMES
|
||||
> 58K
|
||||
> #GoPio
|
||||
> $417.00
|
||||
> 58k @chain
|
||||
> JOIN LBDC STABLECOIN
|
||||
> CHOOSE FREEDOM D&B GANG TO 100K
|
||||
> WENT BACK TO REMOVE 58K FROM THE CHAIN
|
||||
> FACK EVERYTHING
|
||||
> The halvening
|
||||
> 58k forever
|
||||
> don't sell
|
||||
> we're so back
|
||||
> LFG
|
||||
> pump it!!!
|
||||
> 58k forever
|
||||
> EXPECTATIONS
|
||||
> 100k
|
||||
> 58k
|
||||
> bitchcoin reality
|
||||
> U wake up and see the price at 58k
|
||||
> Immediately u buy and stack to Lokepetyfork
|
||||
> 1, U save the raccoons
|
||||
> 2, U raise the price to 59k
|
||||
> #DSB to 100k
|
||||
> 58K MAXI
|
||||
> $580.00
|
||||
> 58K
|
||||
> PLEBS
|
||||
> STACKCHAIN MAGA
|
||||
|
||||
### 12. `img_7333a3df0aac.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 13. `img_7333a3df0aac.jpg` (batch2, error)
|
||||
|
||||
Error: empty content, finish_reason=length
|
||||
|
||||
### 14. `img_7333a3df0aac.jpg` (batch3, transcribed)
|
||||
|
||||
Type: PAGE
|
||||
|
||||
Text:
|
||||
|
||||
> LOKOPETTYFORK
|
||||
> GANG FOREVER
|
||||
> $10.00
|
||||
> YOU ARE PART OF
|
||||
> THE REBEL ALLIANCE,
|
||||
> AND A TRAITOR!
|
||||
> LokoPettyFork
|
||||
> Don't Stop Believing
|
||||
> Bitcoin
|
||||
> $70,206.87
|
||||
> ▲ 3,052.52 (3.01%) Past Year
|
||||
> verizon
|
||||
> MEME
|
||||
> MUSEUM
|
||||
> 100
|
||||
> SAY NO TO 58K CBDC STABLECOIN
|
||||
> CHOOSE FREEDOM ₿SB GANG TO 100K!
|
||||
> BUY BITCOIN
|
||||
> BUY BITCOIN
|
||||
> BUY BITCOIN
|
||||
> BUY BITCOIN
|
||||
> $100.00
|
||||
> @Lokopettyfork
|
||||
> 8722100
|
||||
> WHAT IS YOUR KEY TO SUCCESS?
|
||||
> LOOK, OTHERS BUY MSTR, I JUST STACK TO LOKOPETTYFORK IT'S NOT SOOO COMPLICATED
|
||||
> 100
|
||||
> Lokopettyfork
|
||||
|
||||
### 15. `img_d5ada9cee4e9.jpg` (raw, empty)
|
||||
|
||||
Error: empty transcription in raw source
|
||||
|
||||
### 16. `img_d5ada9cee4e9.jpg` (batch2, error)
|
||||
|
||||
Error: empty content, finish_reason=length
|
||||
|
||||
### 17. `img_d5ada9cee4e9.jpg` (batch3, transcribed)
|
||||
|
||||
Type: PAGE
|
||||
|
||||
Text:
|
||||
|
||||
> DS #GOPIOS
|
||||
> What I think I look like
|
||||
> GO Pios
|
||||
> What I actually look like
|
||||
> Say 'Go Pios'
|
||||
> LEAVE IT ON.
|
||||
> #GOPIOS
|
||||
> Carroll University
|
||||
> Bitcoin Twitter
|
||||
> Me: GOPIOS
|
||||
> #GoPios
|
||||
> #Go Pios
|
||||
> GFY
|
||||
> #GoPios
|
||||
> vechains
|
||||
> Election
|
||||
> 58kgs
|
||||
> Me, just vibing to GO PIOS
|
||||
> GO PIOS
|
||||
> PIZZA? BLOCK? ALL TIME HIGHS?
|
||||
> GO PIOS!
|
||||
> Soul ETF
|
||||
> WEN
|
||||
> GM
|
||||
> GOPIOS
|
||||
> STOP BELIEV
|
||||
> CARROLL
|
||||
> 6
|
||||
|
||||
### 18. `img_fcb2e6a8f066.jpg` (raw, transcribed)
|
||||
|
||||
Type: PHOTO_OF_PERSON_HOLDING_MAGAZINE
|
||||
|
||||
Text:
|
||||
|
||||
> B/H872.372
|
||||
> [unreadable]O CHAIN
|
||||
> [unreadable]INE
|
||||
> [unreadable]OUND 5
|
||||
> MACHINA
|
||||
> SELF-CUSTODY//BIT[unreadable]
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CHECK: re-derive the stackchain catalog from the raw source JSONs and diff
|
||||
against the committed artifact (stackchain_catalog.json + stackchain_catalog.md).
|
||||
|
||||
Exit 0 only if the committed artifact is exactly what the sources produce, plus
|
||||
the markdown summary table is sane. This is a real regeneration test, not a
|
||||
tautology: it re-reads the source files and recomputes every row.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
SRC_RAW = '/root/stackchain_catalog_raw.json'
|
||||
SRC_B2 = '/root/stackchain_catalog_batch2.json'
|
||||
SRC_B3 = '/root/stackchain_catalog_batch3.json'
|
||||
ACTUAL_JSON = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.json'
|
||||
ACTUAL_MD = '/root/troupe-standup/wizard-work/ledger/stackchain_catalog.md'
|
||||
|
||||
TYPE_RE = re.compile(r'"type"\s*:\s*"(.*?)"')
|
||||
TEXT_RE = re.compile(r'"text"\s*:\s*"([\s\S]*?)"')
|
||||
ERROR_RE = re.compile(r'ERROR: (.*?)(?:, full=|$)')
|
||||
|
||||
|
||||
def load(p):
|
||||
with open(p, encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def clean_text(v):
|
||||
return v.replace('\\n', '\n').replace('\\n', '\n')
|
||||
|
||||
|
||||
def parse_entry_value(v):
|
||||
if not isinstance(v, str) or not v.strip():
|
||||
return None
|
||||
try:
|
||||
d = json.loads(v)
|
||||
if isinstance(d, dict):
|
||||
return d
|
||||
except Exception:
|
||||
pass
|
||||
m = TYPE_RE.search(v)
|
||||
text_m = TEXT_RE.search(v)
|
||||
return {
|
||||
'type': m.group(1) if m else '(unknown)',
|
||||
'text': clean_text(text_m.group(1)) if text_m else '',
|
||||
}
|
||||
|
||||
|
||||
def reextract():
|
||||
raw = load(SRC_RAW)
|
||||
b2 = load(SRC_B2)
|
||||
b3 = load(SRC_B3)
|
||||
|
||||
entries = []
|
||||
for idx, (fname, v) in enumerate(raw.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'raw',
|
||||
'status': 'transcribed', 'type': d.get('type') or '(unknown)',
|
||||
'text': d['text'], 'words': len(d['text'].split())})
|
||||
else:
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'raw',
|
||||
'status': 'empty', 'type': '(not transcribed)',
|
||||
'text': '', 'words': 0})
|
||||
|
||||
for idx, (fname, v) in enumerate(b2.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'batch2',
|
||||
'status': 'transcribed', 'type': d.get('type') or '(unknown)',
|
||||
'text': d['text'], 'words': len(d['text'].split())})
|
||||
else:
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'batch2',
|
||||
'status': 'error', 'type': '(retry failed)',
|
||||
'text': '', 'words': 0})
|
||||
|
||||
for idx, (fname, v) in enumerate(b3.items(), 1):
|
||||
d = parse_entry_value(v)
|
||||
if d and d.get('text'):
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'batch3',
|
||||
'status': 'transcribed', 'type': d.get('type') or '(unknown)',
|
||||
'text': d['text'], 'words': len(d['text'].split())})
|
||||
else:
|
||||
entries.append({'no': None, 'filename': fname, 'source': 'batch3',
|
||||
'status': 'error', 'type': '(retry failed)',
|
||||
'text': '', 'words': 0})
|
||||
|
||||
entries.sort(key=lambda e: e['filename'])
|
||||
for i, e in enumerate(entries, 1):
|
||||
e['no'] = i
|
||||
return entries
|
||||
|
||||
|
||||
def load_actual():
|
||||
with open(ACTUAL_JSON, encoding='utf-8') as f:
|
||||
rows = json.load(f)
|
||||
md = open(ACTUAL_MD, encoding='utf-8').read()
|
||||
return rows, md
|
||||
|
||||
|
||||
def md_summary_rows(md):
|
||||
rows = []
|
||||
for line in md.splitlines():
|
||||
if line.startswith('|') and '---' not in line and 'No' not in line:
|
||||
cells = [c.strip() for c in line.strip('|').split('|')]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
errors = []
|
||||
expected = reextract()
|
||||
actual, md = load_actual()
|
||||
|
||||
if not isinstance(actual, list):
|
||||
errors.append('artifact JSON is not a list')
|
||||
actual = []
|
||||
|
||||
if len(actual) != len(expected):
|
||||
errors.append('row count: actual=%d expected=%d' % (len(actual), len(expected)))
|
||||
|
||||
for i, (a, e) in enumerate(zip(actual, expected), 1):
|
||||
for key in ('no', 'filename', 'source', 'status', 'type', 'text', 'words'):
|
||||
if a.get(key) != e[key]:
|
||||
errors.append('row %d field %r: actual=%r expected=%r' % (i, key, a.get(key), e[key]))
|
||||
|
||||
# markdown sanity: every entry must appear in the summary table with matching No/File/Type/Words.
|
||||
seen = set()
|
||||
for cells in md_summary_rows(md):
|
||||
if len(cells) == 6 and cells[1].startswith('`'):
|
||||
try:
|
||||
no = int(cells[0])
|
||||
except ValueError:
|
||||
continue
|
||||
seen.add(cells[1].strip('`'))
|
||||
if no < 1 or no > len(actual):
|
||||
errors.append('md table row No out of range: %s' % no)
|
||||
continue
|
||||
e = actual[no - 1]
|
||||
if cells[2] != e['source'] or cells[3] != e['type'] or cells[5] != str(e['words']):
|
||||
errors.append('md row %d: %r vs %r' % (no, cells[2:], (e['source'], e['type'], e['words'])))
|
||||
expected_files = {r['filename'] for r in expected}
|
||||
if seen != expected_files:
|
||||
errors.append('md table missing/extra files: %s' % sorted(seen ^ expected_files))
|
||||
|
||||
if errors:
|
||||
print('CHECK FAILED:')
|
||||
for e in errors:
|
||||
print(' -', e)
|
||||
sys.exit(1)
|
||||
|
||||
print('CHECK OK: %d rows re-derived from source JSONs and identical to the '
|
||||
'committed artifact.' % len(actual))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user