diff --git a/scripts/build_release.py b/scripts/build_release.py index ee65c19..134ba00 100755 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -72,6 +72,9 @@ DEMO_REQUIRED_CAPTIONS = ( "A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched", "A restored log still says it began as an AI suggestion you corrected — provenance is part of the record", "Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity", + "The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server", + "Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered", + "The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity", "The review tool is a command line, not a screen in the app — this runs it against the committed synthetic sample", "The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits", "The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over", @@ -196,7 +199,10 @@ def main() -> int: run(["python3", "tests/backup-restore.test.py", "-v"], tree) run(["python3", "tests/production-runbook.test.py", "-v"], tree) server_env = dict(os.environ) - server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"}) + # TIMMY_METRICS_TOKEN gates the operator snapshot for this recording only. It is a demo value + # against a loopback-only server, not a credential, and it never appears in an artifact. + server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false", + "TIMMY_METRICS_TOKEN": "demo-metrics-token-not-a-credential"}) # A leftover server from another checkout answers on this port and the readiness probe cannot # tell it apart from ours, so every gate and the recorded demo would silently describe a # different tree. Refuse to build over a live listener, and verify the served bytes afterwards. diff --git a/scripts/record_release_demo.mjs b/scripts/record_release_demo.mjs index 45f9406..4f21c14 100644 --- a/scripts/record_release_demo.mjs +++ b/scripts/record_release_demo.mjs @@ -1537,6 +1537,71 @@ await page.evaluate(node => node.scrollIntoView({ block: 'center' }), await heal await caption('Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity', 3200); await indicate('[data-health-disclosure]'); +// Issue #19, on camera: the operator report is a command line, not a screen in the app. It runs the +// real script against this recording's own server (started with a demo-only loopback token), so every +// number printed on the card was produced by the requests this recording just made — the ingress +// refusals above are counted, not staged. The same script with a wrong token must be refused rather +// than answered, and the token itself must never reach the report. +const operatorToken = process.env.TIMMY_METRICS_TOKEN || ''; +if (!operatorToken) { + throw new Error('the demo server must be started with a demo TIMMY_METRICS_TOKEN so the operator report can be exercised'); +} +async function operatorReport(args) { + return execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', ...args], { cwd: process.cwd() }) + .then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr })) + .catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' })); +} +const reportRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', operatorToken]); +if (reportRun.code !== 0) throw new Error(`the operator report must read the snapshot: ${reportRun.stderr.trim()}`); +const refusedRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', 'demo-token-that-is-wrong']); +if (refusedRun.code !== 1 || !/rejected the operator token/i.test(refusedRun.stderr)) { + throw new Error('a wrong operator token must be refused with a stated reason instead of answered'); +} +if (reportRun.stdout.includes(operatorToken) || refusedRun.stderr.includes('demo-token-that-is-wrong')) { + throw new Error('the operator token must never appear in the report or in an error line'); +} +const parsedRows = []; +let sections = 0; +for (const line of reportRun.stdout.split('\n')) { + const match = /^ {2}(\S.*?)\s{2,}(\S.*)$/.exec(line); + if (match) { parsedRows.push([match[1].trim(), match[2].trim()]); continue; } + const heading = line.trim(); + if (heading === 'APP' || heading === 'QUEUE' || heading === 'MODEL') { + parsedRows.push(['—', '']); + parsedRows.push(['section', heading]); + sections += 1; + } +} +if (sections !== 3) throw new Error(`the operator report must separate the app, queue, and model boundaries, not ${sections} of them`); +const analyzeRow = parsedRows.find(([key]) => /^analyze requests$/i.test(key)); +if (!analyzeRow || Number(analyzeRow[1]) < 1) { + throw new Error(`the report must count this recording's own requests, not a zero: ${reportRun.stdout}`); +} +const metricsCard = await evidenceCard({ + id: 'operator-health-report', + title: 'OPERATOR HEALTH REPORT', + subtitle: 'command line, not app UI — this recording’s own server, demo token', + rows: [ + ['command', 'node scripts/model_metrics_report.mjs'], + ['target', 'http://127.0.0.1:4173 · loopback only'], + ['—', ''], + ...parsedRows, + ['—', ''], + ['wrong token', 'refused · exit 1 · no counters returned'], + ], + font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace', + keyWidth: 74, + gap: '1px', + rowGap: '5px', +}); +await sleep(420); +const metricsCardProof = await evidenceCardProof('operator-health-report'); +if (!metricsCardProof.ok) throw new Error(`Operator report card is not fully readable on screen: ${JSON.stringify(metricsCardProof)}`); +await caption('The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server', 3800); +await caption('Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered', 3600); +await caption('The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity', 4000); +await page.evaluate(() => document.querySelector('#operator-health-report')?.remove()); + await indicate('#delete-all'); await tap('#delete-all', 650); await page.getByText(/Local ledger and photos deleted/).waitFor(); diff --git a/src/model-metrics-report.js b/src/model-metrics-report.js index d55ef12..9c8c0fa 100644 --- a/src/model-metrics-report.js +++ b/src/model-metrics-report.js @@ -6,7 +6,10 @@ // not printed — an unknown failure class is visible as a missing row, never as a new label. import { FAILURE_CLASSES, OUTCOMES } from './model-metrics.js'; -const LABEL_WIDTH = 22; +// Wide enough for the longest row label (`failure provider_unreachable`) plus a visible gap: a label +// that fills the column renders run-together with its count (`failure ingress_refused0`), which reads +// as one broken token instead of a metric. +const LABEL_WIDTH = 30; function row(label, value) { return ` ${String(label).padEnd(LABEL_WIDTH)}${value}`; diff --git a/tests/model-metrics-report-format.test.js b/tests/model-metrics-report-format.test.js index c1fb3f9..5b02d3f 100644 --- a/tests/model-metrics-report-format.test.js +++ b/tests/model-metrics-report-format.test.js @@ -80,3 +80,18 @@ test('the report shows queue occupancy against the configured ceiling', () => { assert.match(report, /in flight\s+2 \/ 3/); assert.match(report, /at capacity\s+no/); }); + +test('every row keeps a visible gap between its label and its count', () => { + const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') }); + // The longest label is the one that used to run together with its value: a column that exactly fits + // the text reads as `failure ingress_refused0`, one broken token instead of a metric and its count. + metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 }); + const report = formatOperatorReport(metrics.snapshot()); + + for (const line of report.split('\n')) { + if (!/^ {2}\S/.test(line)) continue; + assert.match(line, /^ {2}\S.*?\s{2,}\S/, `row has no gap between label and value: ${JSON.stringify(line)}`); + } + assert.match(report, /failure ingress_refused\s+1/); + assert.match(report, /failure provider_unreachable\s+0/); +}); diff --git a/tests/release-demo.test.js b/tests/release-demo.test.js index 818023b..fa316a3 100644 --- a/tests/release-demo.test.js +++ b/tests/release-demo.test.js @@ -575,3 +575,21 @@ test('release demo shows the privacy-safe health counters where the app disclose assert.match(builder, /Health counters are disclosed where you read privacy facts/); assert.match(builder, /"privacy_safe_health_metrics": "passed"/); }); + +test('release demo runs the operator health report against its own server and shows a refused token', async () => { + const demo = await readFile(demoPath, 'utf8'); + const builder = await readFile(builderPath, 'utf8'); + + // The disclosure alone does not demonstrate the feature. The recording must run the real CLI, show + // counters this recording produced, and prove the wrong-token path is refused rather than answered. + assert.match(demo, /scripts\/model_metrics_report\.mjs/); + assert.match(demo, /the operator report must read the snapshot/); + assert.match(demo, /a wrong operator token must be refused with a stated reason instead of answered/); + assert.match(demo, /the report must count this recording's own requests, not a zero/); + assert.match(demo, /const metricsCardProof = await evidenceCardProof\('operator-health-report'\)/); + // The bearer must never be printed on camera or into an error line. + assert.match(demo, /the operator token must never appear in the report or in an error line/); + // The demo server has to carry a token, or the report cannot be exercised at all. + assert.match(builder, /"TIMMY_METRICS_TOKEN": "demo-metrics-token-not-a-credential"/); + assert.match(builder, /The operator health report is a command line, not a screen in the app/); +});