timmy
d9f82299d0
Quality gates / quality (pull_request) Successful in 2m26s
The worker contract was only proven by one happy-path fixture and a handful of inline assertions, so nothing continuously checked that a reply Timmy cannot use is refused instead of guessed at. Corpus: a CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) generated deterministically by scripts/build_vision_corpus.py. Every fixture records its sha256, provenance, licence, case class, the provider reply it replays, and the outcome that reply must produce. loadVisionCorpus refuses a fixture whose bytes no longer match the manifest, so the corpus cannot be edited into a different test. No real medical imagery, camera, or third-party content is involved. Contract fixes the corpus exposed: - A schema-valid reply with a null form and colour is now an abstention (abstainedFor: unusable_visual_fields) instead of a hard provider failure. The declared response schema permits those nulls, so treating the reply as an error was wrong; it now abstains calmly and still prefill nothing. - Fail-closed replies carry a machine-readable code (invalid_json, invalid_fields) instead of prose matching, and visionFailureResponse() maps them to plain, user-safe copy with no provider text, image bytes, or credentials echoed. A running server now returns that copy for unparsable and out-of-range replies. Evidence: unit corpus contract and hash integrity, corpus-wide conformance through the real analysis path, the same corpus through a running server.mjs with a stub worker (200 abstain vs 503 fail-closed, no leaked bytes), and a real browser acceptance run for the refusal UI. The release demo replays three corpus cases through the real code path so the video shows the status the server would actually send, with the on-camera caption gated by the release builder.
285 lines
16 KiB
Python
Executable File
285 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Build a gated, reproducible Timmy review release from committed main."""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT_ROOT = Path(os.environ.get("TIMMY_RELEASE_DIR", "/root/timmy-releases"))
|
|
PUBLIC_REPO = "https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd"
|
|
EXCLUDED_PREFIXES = ("video/", "research/source-pages/", "artifacts/")
|
|
EXCLUDED_SUFFIXES = (".gguf", ".bin", ".safetensors", ".onnx", ".pyc")
|
|
SECRET_PATTERNS = (
|
|
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
|
re.compile(r"(?i)(?:api[_-]?key|access[_-]?token|authorization)\s*[:=]\s*['\"](?!\.\.\.|top-secret|test-|example|placeholder)[A-Za-z0-9_./+:-]{16,}"),
|
|
)
|
|
|
|
# Captions that must be proven on camera (via the demo's own receipt) for the release to pass.
|
|
DEMO_REQUIRED_CAPTIONS = (
|
|
"One clear photo action. Manual logging stays one tap away.",
|
|
"Nothing uploads until explicit consent",
|
|
"The suggestion stays provisional — nothing is applied or saved until you confirm it",
|
|
"Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo",
|
|
"Vision worker offline — the AI and consent steps disappear instead of failing after upload",
|
|
"Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion",
|
|
"Delete all local data clears the ledger in this browser; there is no server copy of it to keep",
|
|
"Hermes Agent connected — free-text, contextual, and server-side",
|
|
"Urgent language is intercepted deterministically before Hermes",
|
|
"An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
|
|
)
|
|
|
|
|
|
def run(args: list[str], cwd: Path, *, capture: bool = False, env: dict[str, str] | None = None) -> str:
|
|
print("+", " ".join(args), flush=True)
|
|
result = subprocess.run(args, cwd=cwd, check=True, text=True, env=env,
|
|
stdout=subprocess.PIPE if capture else None)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def tracked_files(tree: Path) -> list[str]:
|
|
names = run(["git", "ls-files"], tree, capture=True).splitlines()
|
|
return sorted(name for name in names if name and not name.startswith(EXCLUDED_PREFIXES)
|
|
and not name.endswith(EXCLUDED_SUFFIXES))
|
|
|
|
|
|
def scan_secrets(tree: Path, names: list[str]) -> None:
|
|
forbidden_names = [name for name in names if Path(name).name == ".env" or name.endswith((".pem", ".key", ".p12"))]
|
|
findings: list[str] = list(forbidden_names)
|
|
for name in names:
|
|
path = tree / name
|
|
if not path.is_file() or path.stat().st_size > 2_000_000:
|
|
continue
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError:
|
|
continue
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.search(text):
|
|
findings.append(name)
|
|
break
|
|
if findings:
|
|
raise SystemExit("Secret scan failed: " + ", ".join(sorted(set(findings))))
|
|
|
|
|
|
def main() -> int:
|
|
branch = run(["git", "branch", "--show-current"], ROOT, capture=True)
|
|
if branch != "main":
|
|
raise SystemExit(f"Release must build from main, not {branch!r}")
|
|
status = run(["git", "status", "--porcelain"], ROOT, capture=True)
|
|
if status:
|
|
raise SystemExit("Release requires a clean committed tree")
|
|
release_delta = os.environ.get("TIMMY_RELEASE_DELTA",
|
|
"Verified rebuild of the current user-visible feature path — no new feature is claimed.").strip()
|
|
if not release_delta:
|
|
raise SystemExit("TIMMY_RELEASE_DELTA must not be empty")
|
|
commit = run(["git", "rev-parse", "HEAD"], ROOT, capture=True)
|
|
short = commit[:12]
|
|
commit_epoch = int(run(["git", "show", "-s", "--format=%ct", commit], ROOT, capture=True))
|
|
source_date = dt.datetime.fromtimestamp(commit_epoch, dt.timezone.utc)
|
|
release_date = dt.datetime.now(dt.timezone.utc).date().isoformat()
|
|
version = f"{release_date}-{short}"
|
|
release_dir = OUT_ROOT / version
|
|
if release_dir.exists():
|
|
shutil.rmtree(release_dir)
|
|
release_dir.mkdir(parents=True)
|
|
|
|
with tempfile.TemporaryDirectory(prefix="timmy-release-") as tmp:
|
|
tree = Path(tmp) / "source"
|
|
run(["git", "clone", "--quiet", "--no-local", str(ROOT), str(tree)], ROOT)
|
|
run(["git", "checkout", "--quiet", commit], tree)
|
|
run(["npm", "ci", "--ignore-scripts"], tree)
|
|
run(["npm", "test"], tree)
|
|
# The committed corpus hashes are an integrity receipt: verify them before anything is built.
|
|
run(["python3", "scripts/build_vision_corpus.py", "--verify"], tree)
|
|
server_env = dict(os.environ)
|
|
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
|
|
server_log = (release_dir / "acceptance-server.log").open("w", encoding="utf-8")
|
|
server = subprocess.Popen(["node", "server.mjs"], cwd=tree, env=server_env,
|
|
stdout=server_log, stderr=subprocess.STDOUT, text=True)
|
|
try:
|
|
for _ in range(50):
|
|
if server.poll() is not None:
|
|
raise SystemExit("Acceptance server exited before readiness")
|
|
try:
|
|
urllib.request.urlopen("http://127.0.0.1:4173/", timeout=1).read(1)
|
|
break
|
|
except Exception:
|
|
time.sleep(0.1)
|
|
else:
|
|
raise SystemExit("Acceptance server did not become ready")
|
|
run(["npm", "run", "test:ui"], tree)
|
|
run(["npm", "run", "test:photo"], tree)
|
|
run(["npm", "run", "test:mobile-capture"], tree)
|
|
run(["npm", "run", "test:sleek"], tree)
|
|
run(["npm", "run", "test:manual-fallback"], tree)
|
|
run(["npm", "run", "test:provisional"], tree)
|
|
run(["npm", "run", "test:followup"], tree)
|
|
run(["npm", "run", "test:vision-refusal"], tree)
|
|
demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
|
|
caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
|
|
demo_env = dict(server_env)
|
|
demo_env["TIMMY_RELEASE_VERSION"] = version
|
|
demo_env["TIMMY_RELEASE_DEMO_DELTA"] = release_delta
|
|
demo_env["TIMMY_DEMO_CAPTIONS_OUT"] = str(caption_receipt)
|
|
run(["node", "scripts/record_release_demo.mjs", str(demo_raw)], tree, env=demo_env)
|
|
if not caption_receipt.exists():
|
|
raise SystemExit("Release demo produced no on-camera caption receipt")
|
|
recorded = json.loads(caption_receipt.read_text(encoding="utf-8"))
|
|
onscreen = [str(line) for line in (recorded.get("captions") or [])]
|
|
if recorded.get("releaseDelta") != release_delta or not any(release_delta in line for line in onscreen):
|
|
raise SystemExit("Release demo did not state the declared release delta on camera")
|
|
for required in DEMO_REQUIRED_CAPTIONS:
|
|
if not any(required in line for line in onscreen):
|
|
raise SystemExit(f"Release demo is missing a required on-camera caption: {required}")
|
|
finally:
|
|
server.terminate()
|
|
try:
|
|
server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
server.kill()
|
|
server.wait(timeout=5)
|
|
server_log.close()
|
|
demo = release_dir / f"timmy-talking-turd-{version}-demo.mp4"
|
|
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo_raw), "-vf", "scale=720:1280:flags=lanczos", "-an", "-c:v", "libx264", "-preset", "fast", "-crf", "20", "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(demo)], tree)
|
|
demo_raw.unlink()
|
|
run(["ffmpeg", "-v", "error", "-i", str(demo), "-f", "null", "-"], tree)
|
|
probe = json.loads(run(["ffprobe", "-v", "error", "-print_format", "json", "-show_streams", "-show_format", str(demo)], tree, capture=True))
|
|
video_streams = [stream for stream in probe.get("streams", []) if stream.get("codec_type") == "video"]
|
|
if len(video_streams) != 1 or video_streams[0].get("codec_name") != "h264" or video_streams[0].get("pix_fmt") != "yuv420p" or video_streams[0].get("width") != 720 or video_streams[0].get("height") != 1280:
|
|
raise SystemExit("Release demo media probe failed")
|
|
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
|
# Span the whole recording: a fixed 1/2 fps sheet only covered the first 24s of a longer demo.
|
|
demo_duration = float(run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(demo)], tree, capture=True))
|
|
if demo_duration <= 0:
|
|
raise SystemExit("Release demo has no measurable duration")
|
|
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", f"fps={12 / demo_duration:.6f},scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
|
if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
|
|
raise SystemExit("Release demo contact sheet was not produced")
|
|
run(["npm", "audit", "--audit-level=high"], tree)
|
|
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"):
|
|
run(["node", "--check", file], tree)
|
|
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
|
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
|
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree)
|
|
run(["git", "diff", "--check", commit], tree)
|
|
|
|
names = tracked_files(tree)
|
|
scan_secrets(tree, names)
|
|
if any(name.startswith(EXCLUDED_PREFIXES) or name.endswith(EXCLUDED_SUFFIXES) for name in names):
|
|
raise SystemExit("Release allowlist included a forbidden artifact")
|
|
|
|
package_root = Path(tmp) / f"timmy-talking-turd-{version}"
|
|
for name in names:
|
|
src = tree / name
|
|
dst = package_root / name
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
|
|
notes = (
|
|
f"# Timmy review release {version}\n\n"
|
|
f"- Source commit: [{commit}]({PUBLIC_REPO}/commit/{commit})\n"
|
|
f"- Build date (UTC): {release_date}\n"
|
|
f"- Release delta: {release_delta}\n"
|
|
"- Sleek three-destination shell with one dominant photo action and quiet manual fallback\n"
|
|
"- Manual fallback: vision-offline logging stays one action away with an in-session draft and explicit start over\n"
|
|
"- Smart free-text Timmy chat backed by authenticated server-side Hermes session continuity\n"
|
|
"- Agent safety: exact-origin HttpOnly session, bounded text-only ledger context, no browser credentials or session IDs\n"
|
|
"- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n"
|
|
"- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
|
|
"- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
|
|
"- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied\n"
|
|
"- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
|
|
"- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
|
|
"- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
|
|
)
|
|
(package_root / "RELEASE-NOTES.md").write_text(notes, encoding="utf-8")
|
|
|
|
archive = release_dir / f"timmy-talking-turd-{version}.tar.gz"
|
|
with archive.open("wb") as raw:
|
|
import gzip
|
|
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=commit_epoch) as gz:
|
|
with tarfile.open(fileobj=gz, mode="w") as tar:
|
|
for path in sorted(package_root.rglob("*")):
|
|
arcname = path.relative_to(package_root.parent)
|
|
info = tar.gettarinfo(str(path), arcname=str(arcname))
|
|
info.uid = info.gid = 0
|
|
info.uname = info.gname = ""
|
|
info.mtime = commit_epoch
|
|
if path.is_file():
|
|
with path.open("rb") as handle:
|
|
tar.addfile(info, handle)
|
|
else:
|
|
tar.addfile(info)
|
|
|
|
digest = sha256(archive)
|
|
demo_digest = sha256(demo)
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"project": "Timmy the Talking Turd",
|
|
"version": version,
|
|
"commit": commit,
|
|
"release_delta": release_delta,
|
|
"source_date_utc": source_date.isoformat(),
|
|
"built_at_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"artifact": archive.name,
|
|
"bytes": archive.stat().st_size,
|
|
"sha256": digest,
|
|
"feature_demo": {
|
|
"artifact": demo.name,
|
|
"bytes": demo.stat().st_size,
|
|
"sha256": demo_digest,
|
|
"width": 720,
|
|
"height": 1280,
|
|
"codec": "h264",
|
|
"pixel_format": "yuv420p",
|
|
"recorded_from_working_app": True,
|
|
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, and an out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response",
|
|
},
|
|
"gates": {
|
|
"unit_security": "passed",
|
|
"mobile_green_path": "passed",
|
|
"photo_first_acceptance": "passed",
|
|
"sleek_hermes_chat_acceptance": "passed",
|
|
"manual_fallback_draft_recovery": "passed",
|
|
"provisional_suggestion_state": "passed",
|
|
"hard_stool_followup": "passed",
|
|
"vision_corpus_refusal": "passed",
|
|
"dependency_audit_high": "passed",
|
|
"syntax": "passed",
|
|
"secret_scan": "passed",
|
|
"forbidden_artifacts": "passed",
|
|
"feature_demo_full_decode": "passed",
|
|
"feature_demo_media_probe": "passed",
|
|
"release_delta_visible_on_camera": "passed",
|
|
},
|
|
}
|
|
manifest_path = release_dir / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
(release_dir / "SHA256SUMS").write_text(f"{digest} {archive.name}\n{demo_digest} {demo.name}\n", encoding="utf-8")
|
|
print(json.dumps({"release_dir": str(release_dir), "archive": str(archive), "feature_demo": str(demo), "manifest": str(manifest_path), "sha256": digest, "feature_demo_sha256": demo_digest}, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|