timmy
5c759078c4
Quality gates / quality (pull_request) Failing after 14s
The blank-first ordering let one blank field short-circuit the whole reply, so a response carrying an impossible value came back as `unusable_visual_fields`: the user was told the photo was unreadable when the model had in fact returned something Timmy must never apply. Blank fields still abstain and still prefill nothing. A filled-in field is a claim, and a claim outside the contract now fails closed even when the other field came back blank. The corpus earns its label too. `out-of-range-fields` drops from confidence 1.4 to 0.9 so the case reaches the field-range check instead of dying at the confidence gate, and a new `blank-plus-out-of-range` case covers the class the old ordering let through. The release demo replays that case on camera and the fail-closed 503 budget moves from one to two, so the scene cannot be dropped silently. Closes #73
286 lines
17 KiB
Python
Executable File
286 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Build a gated, reproducible Timmy review release from committed main."""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT_ROOT = Path(os.environ.get("TIMMY_RELEASE_DIR", "/root/timmy-releases"))
|
|
PUBLIC_REPO = "https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd"
|
|
EXCLUDED_PREFIXES = ("video/", "research/source-pages/", "artifacts/")
|
|
EXCLUDED_SUFFIXES = (".gguf", ".bin", ".safetensors", ".onnx", ".pyc")
|
|
SECRET_PATTERNS = (
|
|
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
|
re.compile(r"(?i)(?:api[_-]?key|access[_-]?token|authorization)\s*[:=]\s*['\"](?!\.\.\.|top-secret|test-|example|placeholder)[A-Za-z0-9_./+:-]{16,}"),
|
|
)
|
|
|
|
# Captions that must be proven on camera (via the demo's own receipt) for the release to pass.
|
|
DEMO_REQUIRED_CAPTIONS = (
|
|
"One clear photo action. Manual logging stays one tap away.",
|
|
"Nothing uploads until explicit consent",
|
|
"The suggestion stays provisional — nothing is applied or saved until you confirm it",
|
|
"Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo",
|
|
"Vision worker offline — the AI and consent steps disappear instead of failing after upload",
|
|
"Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion",
|
|
"Delete all local data clears the ledger in this browser; there is no server copy of it to keep",
|
|
"Hermes Agent connected — free-text, contextual, and server-side",
|
|
"Urgent language is intercepted deterministically before Hermes",
|
|
"An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
|
|
"A filled-in field that is out of range still fails closed, even when the other field came back blank",
|
|
)
|
|
|
|
|
|
def run(args: list[str], cwd: Path, *, capture: bool = False, env: dict[str, str] | None = None) -> str:
|
|
print("+", " ".join(args), flush=True)
|
|
result = subprocess.run(args, cwd=cwd, check=True, text=True, env=env,
|
|
stdout=subprocess.PIPE if capture else None)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def tracked_files(tree: Path) -> list[str]:
|
|
names = run(["git", "ls-files"], tree, capture=True).splitlines()
|
|
return sorted(name for name in names if name and not name.startswith(EXCLUDED_PREFIXES)
|
|
and not name.endswith(EXCLUDED_SUFFIXES))
|
|
|
|
|
|
def scan_secrets(tree: Path, names: list[str]) -> None:
|
|
forbidden_names = [name for name in names if Path(name).name == ".env" or name.endswith((".pem", ".key", ".p12"))]
|
|
findings: list[str] = list(forbidden_names)
|
|
for name in names:
|
|
path = tree / name
|
|
if not path.is_file() or path.stat().st_size > 2_000_000:
|
|
continue
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError:
|
|
continue
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.search(text):
|
|
findings.append(name)
|
|
break
|
|
if findings:
|
|
raise SystemExit("Secret scan failed: " + ", ".join(sorted(set(findings))))
|
|
|
|
|
|
def main() -> int:
|
|
branch = run(["git", "branch", "--show-current"], ROOT, capture=True)
|
|
if branch != "main":
|
|
raise SystemExit(f"Release must build from main, not {branch!r}")
|
|
status = run(["git", "status", "--porcelain"], ROOT, capture=True)
|
|
if status:
|
|
raise SystemExit("Release requires a clean committed tree")
|
|
release_delta = os.environ.get("TIMMY_RELEASE_DELTA",
|
|
"Verified rebuild of the current user-visible feature path — no new feature is claimed.").strip()
|
|
if not release_delta:
|
|
raise SystemExit("TIMMY_RELEASE_DELTA must not be empty")
|
|
commit = run(["git", "rev-parse", "HEAD"], ROOT, capture=True)
|
|
short = commit[:12]
|
|
commit_epoch = int(run(["git", "show", "-s", "--format=%ct", commit], ROOT, capture=True))
|
|
source_date = dt.datetime.fromtimestamp(commit_epoch, dt.timezone.utc)
|
|
release_date = dt.datetime.now(dt.timezone.utc).date().isoformat()
|
|
version = f"{release_date}-{short}"
|
|
release_dir = OUT_ROOT / version
|
|
if release_dir.exists():
|
|
shutil.rmtree(release_dir)
|
|
release_dir.mkdir(parents=True)
|
|
|
|
with tempfile.TemporaryDirectory(prefix="timmy-release-") as tmp:
|
|
tree = Path(tmp) / "source"
|
|
run(["git", "clone", "--quiet", "--no-local", str(ROOT), str(tree)], ROOT)
|
|
run(["git", "checkout", "--quiet", commit], tree)
|
|
run(["npm", "ci", "--ignore-scripts"], tree)
|
|
run(["npm", "test"], tree)
|
|
# The committed corpus hashes are an integrity receipt: verify them before anything is built.
|
|
run(["python3", "scripts/build_vision_corpus.py", "--verify"], tree)
|
|
server_env = dict(os.environ)
|
|
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
|
|
server_log = (release_dir / "acceptance-server.log").open("w", encoding="utf-8")
|
|
server = subprocess.Popen(["node", "server.mjs"], cwd=tree, env=server_env,
|
|
stdout=server_log, stderr=subprocess.STDOUT, text=True)
|
|
try:
|
|
for _ in range(50):
|
|
if server.poll() is not None:
|
|
raise SystemExit("Acceptance server exited before readiness")
|
|
try:
|
|
urllib.request.urlopen("http://127.0.0.1:4173/", timeout=1).read(1)
|
|
break
|
|
except Exception:
|
|
time.sleep(0.1)
|
|
else:
|
|
raise SystemExit("Acceptance server did not become ready")
|
|
run(["npm", "run", "test:ui"], tree)
|
|
run(["npm", "run", "test:photo"], tree)
|
|
run(["npm", "run", "test:mobile-capture"], tree)
|
|
run(["npm", "run", "test:sleek"], tree)
|
|
run(["npm", "run", "test:manual-fallback"], tree)
|
|
run(["npm", "run", "test:provisional"], tree)
|
|
run(["npm", "run", "test:followup"], tree)
|
|
run(["npm", "run", "test:vision-refusal"], tree)
|
|
demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
|
|
caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
|
|
demo_env = dict(server_env)
|
|
demo_env["TIMMY_RELEASE_VERSION"] = version
|
|
demo_env["TIMMY_RELEASE_DEMO_DELTA"] = release_delta
|
|
demo_env["TIMMY_DEMO_CAPTIONS_OUT"] = str(caption_receipt)
|
|
run(["node", "scripts/record_release_demo.mjs", str(demo_raw)], tree, env=demo_env)
|
|
if not caption_receipt.exists():
|
|
raise SystemExit("Release demo produced no on-camera caption receipt")
|
|
recorded = json.loads(caption_receipt.read_text(encoding="utf-8"))
|
|
onscreen = [str(line) for line in (recorded.get("captions") or [])]
|
|
if recorded.get("releaseDelta") != release_delta or not any(release_delta in line for line in onscreen):
|
|
raise SystemExit("Release demo did not state the declared release delta on camera")
|
|
for required in DEMO_REQUIRED_CAPTIONS:
|
|
if not any(required in line for line in onscreen):
|
|
raise SystemExit(f"Release demo is missing a required on-camera caption: {required}")
|
|
finally:
|
|
server.terminate()
|
|
try:
|
|
server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
server.kill()
|
|
server.wait(timeout=5)
|
|
server_log.close()
|
|
demo = release_dir / f"timmy-talking-turd-{version}-demo.mp4"
|
|
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo_raw), "-vf", "scale=720:1280:flags=lanczos", "-an", "-c:v", "libx264", "-preset", "fast", "-crf", "20", "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(demo)], tree)
|
|
demo_raw.unlink()
|
|
run(["ffmpeg", "-v", "error", "-i", str(demo), "-f", "null", "-"], tree)
|
|
probe = json.loads(run(["ffprobe", "-v", "error", "-print_format", "json", "-show_streams", "-show_format", str(demo)], tree, capture=True))
|
|
video_streams = [stream for stream in probe.get("streams", []) if stream.get("codec_type") == "video"]
|
|
if len(video_streams) != 1 or video_streams[0].get("codec_name") != "h264" or video_streams[0].get("pix_fmt") != "yuv420p" or video_streams[0].get("width") != 720 or video_streams[0].get("height") != 1280:
|
|
raise SystemExit("Release demo media probe failed")
|
|
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
|
# Span the whole recording: a fixed 1/2 fps sheet only covered the first 24s of a longer demo.
|
|
demo_duration = float(run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(demo)], tree, capture=True))
|
|
if demo_duration <= 0:
|
|
raise SystemExit("Release demo has no measurable duration")
|
|
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", f"fps={12 / demo_duration:.6f},scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
|
if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
|
|
raise SystemExit("Release demo contact sheet was not produced")
|
|
run(["npm", "audit", "--audit-level=high"], tree)
|
|
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"):
|
|
run(["node", "--check", file], tree)
|
|
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
|
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
|
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree)
|
|
run(["git", "diff", "--check", commit], tree)
|
|
|
|
names = tracked_files(tree)
|
|
scan_secrets(tree, names)
|
|
if any(name.startswith(EXCLUDED_PREFIXES) or name.endswith(EXCLUDED_SUFFIXES) for name in names):
|
|
raise SystemExit("Release allowlist included a forbidden artifact")
|
|
|
|
package_root = Path(tmp) / f"timmy-talking-turd-{version}"
|
|
for name in names:
|
|
src = tree / name
|
|
dst = package_root / name
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
|
|
notes = (
|
|
f"# Timmy review release {version}\n\n"
|
|
f"- Source commit: [{commit}]({PUBLIC_REPO}/commit/{commit})\n"
|
|
f"- Build date (UTC): {release_date}\n"
|
|
f"- Release delta: {release_delta}\n"
|
|
"- Sleek three-destination shell with one dominant photo action and quiet manual fallback\n"
|
|
"- Manual fallback: vision-offline logging stays one action away with an in-session draft and explicit start over\n"
|
|
"- Smart free-text Timmy chat backed by authenticated server-side Hermes session continuity\n"
|
|
"- Agent safety: exact-origin HttpOnly session, bounded text-only ledger context, no browser credentials or session IDs\n"
|
|
"- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n"
|
|
"- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
|
|
"- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
|
|
"- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied — including a reply that fills in one field and leaves the other blank, where an out-of-range value still fails closed instead of being reported as an unreadable photo (issue #73)\n"
|
|
"- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
|
|
"- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
|
|
"- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
|
|
)
|
|
(package_root / "RELEASE-NOTES.md").write_text(notes, encoding="utf-8")
|
|
|
|
archive = release_dir / f"timmy-talking-turd-{version}.tar.gz"
|
|
with archive.open("wb") as raw:
|
|
import gzip
|
|
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=commit_epoch) as gz:
|
|
with tarfile.open(fileobj=gz, mode="w") as tar:
|
|
for path in sorted(package_root.rglob("*")):
|
|
arcname = path.relative_to(package_root.parent)
|
|
info = tar.gettarinfo(str(path), arcname=str(arcname))
|
|
info.uid = info.gid = 0
|
|
info.uname = info.gname = ""
|
|
info.mtime = commit_epoch
|
|
if path.is_file():
|
|
with path.open("rb") as handle:
|
|
tar.addfile(info, handle)
|
|
else:
|
|
tar.addfile(info)
|
|
|
|
digest = sha256(archive)
|
|
demo_digest = sha256(demo)
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"project": "Timmy the Talking Turd",
|
|
"version": version,
|
|
"commit": commit,
|
|
"release_delta": release_delta,
|
|
"source_date_utc": source_date.isoformat(),
|
|
"built_at_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"artifact": archive.name,
|
|
"bytes": archive.stat().st_size,
|
|
"sha256": digest,
|
|
"feature_demo": {
|
|
"artifact": demo.name,
|
|
"bytes": demo.stat().st_size,
|
|
"sha256": demo_digest,
|
|
"width": 720,
|
|
"height": 1280,
|
|
"codec": "h264",
|
|
"pixel_format": "yuv420p",
|
|
"recorded_from_working_app": True,
|
|
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response",
|
|
},
|
|
"gates": {
|
|
"unit_security": "passed",
|
|
"mobile_green_path": "passed",
|
|
"photo_first_acceptance": "passed",
|
|
"sleek_hermes_chat_acceptance": "passed",
|
|
"manual_fallback_draft_recovery": "passed",
|
|
"provisional_suggestion_state": "passed",
|
|
"hard_stool_followup": "passed",
|
|
"vision_corpus_refusal": "passed",
|
|
"dependency_audit_high": "passed",
|
|
"syntax": "passed",
|
|
"secret_scan": "passed",
|
|
"forbidden_artifacts": "passed",
|
|
"feature_demo_full_decode": "passed",
|
|
"feature_demo_media_probe": "passed",
|
|
"release_delta_visible_on_camera": "passed",
|
|
},
|
|
}
|
|
manifest_path = release_dir / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
(release_dir / "SHA256SUMS").write_text(f"{digest} {archive.name}\n{demo_digest} {demo.name}\n", encoding="utf-8")
|
|
print(json.dumps({"release_dir": str(release_dir), "archive": str(archive), "feature_demo": str(demo), "manifest": str(manifest_path), "sha256": digest, "feature_demo_sha256": demo_digest}, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|