#!/usr/bin/env python3 """Behavior-preservation proof for the pdf_create / pdf_make_form refactor. Compares the BACKED-UP original (before/) against the refactored in-place script (skills/.../) with identical inputs. The refactor only touches two code paths: stdio reconfiguration and JSON emission (both live in _pdf_common). PDF generation bodies are untouched, so we additionally assert page-count + extracted text equivalence and byte-identity of the normalized PDF. The JSON payload embeds the --output path, so both runs are pointed at the SAME output-path string (run sequentially; each PDF is copied aside before the next run overwrites it). This makes the embedded `output` field identical across runs, yielding a fair byte-level stdout comparison. reportlab embeds a per-run /CreationDate timestamp, so raw PDF bytes differ even for identical specs; we normalize by stripping volatile Info keys before hashing. FALSE-GREEN GUARD (the defect this harness previously had): reportlab and pypdf must be importable in the interpreter running the SUT runs, otherwise BOTH runs fail identically at import with exit code 2, stdout is empty for both, and the "stdout byte-identical" gate degenerates to empty==empty -> a false PASS. To eliminate that, on startup we: 1. confirm `sys.executable` (PY) can import reportlab+pypdf; 2. if not, re-exec ourselves under the first discovered hermes tools python3.14 that can import both; 3. if none is found, exit non-zero with a clear message (never PASS). Exits 0 only if every hard gate passes. """ from __future__ import annotations import glob import json import os import shutil import subprocess import sys import tempfile from pathlib import Path CREATE_ORIG = Path("/root/.hermes/wizard-outputs/forge/before/pdf_create.py") CREATE_REF = Path("/root/.hermes/profiles/forge/skills/productivity/pdf/scripts/pdf_create.py") FORM_ORIG = Path("/root/.hermes/wizard-outputs/forge/before/pdf_make_form.py") FORM_REF = Path("/root/.hermes/profiles/forge/skills/productivity/pdf/scripts/pdf_make_form.py") CREATE_SPEC = { "title": "Ref Test", "author": "forge", "page_size": "A4", "page_numbers": True, "elements": [ {"type": "heading", "text": "Section 1", "level": 1}, {"type": "paragraph", "text": "Hello world."}, {"type": "table", "rows": [["H1", "H2"], ["a", "b"]], "header": True}, {"type": "pagebreak"}, {"type": "paragraph", "text": "Second page text."}, ], } FORM_SPEC = { "title": "Intake", "author": "forge", "page_size": "A4", "page_count": 1, "fields": [ {"name": "surname", "type": "text", "page": 1, "label": "Surname", "label_box": [72, 700, 150, 714], "entry_box": [160, 696, 400, 716], "value": "Smith", "tooltip": "Family name"}, {"name": "agree", "type": "checkbox", "page": 1, "label": "I agree", "label_box": [72, 660, 150, 674], "entry_box": [160, 658, 176, 674], "checked": True}, {"name": "color", "type": "radio", "page": 1, "label": "Color", "label_box": [72, 620, 150, 634], "entry_box": [160, 616, 400, 636], "options": ["red", "blue"], "value": "red"}, {"name": "size", "type": "dropdown", "page": 1, "label": "Size", "label_box": [72, 580, 150, 594], "entry_box": [160, 576, 300, 596], "options": ["small", "large"], "value": "small"}, ], } def _has_deps(interp: str) -> bool: """True iff *interp* can import reportlab and pypdf (the SUT's runtime deps).""" return subprocess.run( [interp, "-c", "import reportlab, pypdf"], capture_output=True, ).returncode == 0 def _ensure_capable_interpreter() -> None: """Re-exec under an interpreter that has reportlab+pypdf, or fail loudly. This is the fix for the prior false-green: without it, launching under an interpreter lacking the deps made every SUT run fail at import (exit 2) and every gate a vacuous truth. """ if _has_deps(sys.executable): return candidates = sorted(glob.glob("/root/.hermes/tools/python-*linux-x64/bin/python3.14")) for c in candidates: if _has_deps(c): print(f"[guard] re-execing under {c} (current {sys.executable} lacks reportlab/pypdf)", file=sys.stderr) os.execv(c, [c, *sys.argv]) print("[guard] ERROR: no interpreter with reportlab+pypdf found " "(cannot run SUT gates). Aborting instead of false-greening.", file=sys.stderr) sys.exit(2) _ensure_capable_interpreter() PY = sys.executable # now guaranteed to import reportlab+pypdf def _run(script: Path, spec_path: Path, out_pdf: Path): return subprocess.run( [PY, str(script), str(spec_path), "-o", str(out_pdf)], capture_output=True, text=True, timeout=120, ) def normalize_pdf_bytes(path: Path) -> bytes: """Re-serialize via pypdf and strip volatile Info keys -> deterministic bytes.""" from pypdf import PdfReader, PdfWriter r = PdfReader(str(path)) w = PdfWriter() w.append(r) info = w._root_object["/Info"] if "/Info" in w._root_object else None if info is not None: for k in ("/CreationDate", "/ModDate", "/Producer", "/Creator"): try: del info[k] except KeyError: pass fd, tmp = tempfile.mkstemp(suffix=".pdf") os.close(fd) try: with open(tmp, "wb") as f: w.write(f) return Path(tmp).read_bytes() finally: Path(tmp).unlink(missing_ok=True) def pdf_text_and_pages(path: Path): from pypdf import PdfReader r = PdfReader(str(path)) pages = len(r.pages) text = "".join((p.extract_text() or "") for p in r.pages) return pages, text def check(label, orig_script, ref_script, spec): work = Path(tempfile.mkdtemp()) spec_path = work / "spec.json" spec_path.write_text(json.dumps(spec), encoding="utf-8") out_pdf = work / "result.pdf" # SAME path string used by both runs or_ = _run(orig_script, spec_path, out_pdf) orig_pdf = work / "orig.pdf" if out_pdf.exists(): shutil.copyfile(out_pdf, orig_pdf) out_pdf.unlink() rr = _run(ref_script, spec_path, out_pdf) ref_pdf = work / "ref.pdf" if out_pdf.exists(): shutil.copyfile(out_pdf, ref_pdf) out_pdf.unlink() gates = [ ("stdout byte-identical", or_.stdout == rr.stdout), ("exit code identical", or_.returncode == rr.returncode), ("stderr identical", or_.stderr == rr.stderr), ] # HARD gate: the SUT must actually run its PDF path, not fail at import. if or_.returncode == 2 or rr.returncode == 2: gates.append(("sut ran real path (not import-error exit 2)", False)) else: gates.append(("sut ran real path (not import-error exit 2)", True)) notes = [] if or_.returncode == 0 and rr.returncode == 0 and orig_pdf.exists() and ref_pdf.exists(): op, ot = pdf_text_and_pages(orig_pdf) rp, rt = pdf_text_and_pages(ref_pdf) gates.append(("pdf page count identical", op == rp)) gates.append(("pdf extracted text identical", ot == rt)) nb_eq = normalize_pdf_bytes(orig_pdf) == normalize_pdf_bytes(ref_pdf) gates.append(("normalized pdf bytes identical", nb_eq)) notes.append(f"pages={op}/{rp} text_eq={ot==rt} norm_bytes_eq={nb_eq}") else: gates.append(("dep-present runs to completion (exit 0)", False)) print(f"=== {label} ===") for name, ok in gates: print(f" [{'PASS' if ok else 'FAIL'}] {name}") print(f" stdout_len orig={len(or_.stdout)} ref={len(rr.stdout)} | " f"exit {or_.returncode}/{rr.returncode} | stderr_len {len(or_.stderr)}/{len(rr.stderr)}") if notes: print(f" pdf: {' '.join(notes)}") ok_exit2 = or_.returncode == 2 or rr.returncode == 2 all_ok = all(ok for _, ok in gates) and not ok_exit2 print(f" -> {'PASS' if all_ok else 'FAIL'}") return all_ok def main() -> int: ok = True ok &= check("pdf_create.py", CREATE_ORIG, CREATE_REF, CREATE_SPEC) ok &= check("pdf_make_form.py", FORM_ORIG, FORM_REF, FORM_SPEC) print("\nSUMMARY:", "ALL PASS" if ok else "FAILURES DETECTED") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())