feat(demo): run the operator health report on camera and fix the run-together row
Quality gates / quality (push) Canceled after 0s
Quality gates / quality (push) Canceled after 0s
The release video proved the disclosure sentence but never exercised the operator surface it describes. This records the real scripts/model_metrics_report.mjs against the recording's own server, shows the counters that run produced, and proves a wrong token is refused instead of answered. The first run of that CLI printed 'failure ingress_refused0': the longest failure label exactly filled the 22-character column, so label and count rendered as one broken token. LABEL_WIDTH is now 30, and a format test fails if any row loses the gap between its label and its value.
This commit is contained in:
1 parent
48979bf744
commit
ef83ad93bb
5 files changed
+109
-2
No files matched your search
@@ -72,6 +72,9 @@ DEMO_REQUIRED_CAPTIONS = (
|
||||
"A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched",
|
||||
"A restored log still says it began as an AI suggestion you corrected — provenance is part of the record",
|
||||
"Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity",
|
||||
"The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server",
|
||||
"Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered",
|
||||
"The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity",
|
||||
"The review tool is a command line, not a screen in the app — this runs it against the committed synthetic sample",
|
||||
"The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits",
|
||||
"The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over",
|
||||
@@ -196,7 +199,10 @@ def main() -> int:
|
||||
run(["python3", "tests/backup-restore.test.py", "-v"], tree)
|
||||
run(["python3", "tests/production-runbook.test.py", "-v"], tree)
|
||||
server_env = dict(os.environ)
|
||||
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
|
||||
# TIMMY_METRICS_TOKEN gates the operator snapshot for this recording only. It is a demo value
|
||||
# against a loopback-only server, not a credential, and it never appears in an artifact.
|
||||
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false",
|
||||
"TIMMY_METRICS_TOKEN": "demo-metrics-token-not-a-credential"})
|
||||
# A leftover server from another checkout answers on this port and the readiness probe cannot
|
||||
# tell it apart from ours, so every gate and the recorded demo would silently describe a
|
||||
# different tree. Refuse to build over a live listener, and verify the served bytes afterwards.
|
||||
|
||||
@@ -1537,6 +1537,71 @@ await page.evaluate(node => node.scrollIntoView({ block: 'center' }), await heal
|
||||
await caption('Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity', 3200);
|
||||
await indicate('[data-health-disclosure]');
|
||||
|
||||
// Issue #19, on camera: the operator report is a command line, not a screen in the app. It runs the
|
||||
// real script against this recording's own server (started with a demo-only loopback token), so every
|
||||
// number printed on the card was produced by the requests this recording just made — the ingress
|
||||
// refusals above are counted, not staged. The same script with a wrong token must be refused rather
|
||||
// than answered, and the token itself must never reach the report.
|
||||
const operatorToken = process.env.TIMMY_METRICS_TOKEN || '';
|
||||
if (!operatorToken) {
|
||||
throw new Error('the demo server must be started with a demo TIMMY_METRICS_TOKEN so the operator report can be exercised');
|
||||
}
|
||||
async function operatorReport(args) {
|
||||
return execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', ...args], { cwd: process.cwd() })
|
||||
.then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr }))
|
||||
.catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' }));
|
||||
}
|
||||
const reportRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', operatorToken]);
|
||||
if (reportRun.code !== 0) throw new Error(`the operator report must read the snapshot: ${reportRun.stderr.trim()}`);
|
||||
const refusedRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', 'demo-token-that-is-wrong']);
|
||||
if (refusedRun.code !== 1 || !/rejected the operator token/i.test(refusedRun.stderr)) {
|
||||
throw new Error('a wrong operator token must be refused with a stated reason instead of answered');
|
||||
}
|
||||
if (reportRun.stdout.includes(operatorToken) || refusedRun.stderr.includes('demo-token-that-is-wrong')) {
|
||||
throw new Error('the operator token must never appear in the report or in an error line');
|
||||
}
|
||||
const parsedRows = [];
|
||||
let sections = 0;
|
||||
for (const line of reportRun.stdout.split('\n')) {
|
||||
const match = /^ {2}(\S.*?)\s{2,}(\S.*)$/.exec(line);
|
||||
if (match) { parsedRows.push([match[1].trim(), match[2].trim()]); continue; }
|
||||
const heading = line.trim();
|
||||
if (heading === 'APP' || heading === 'QUEUE' || heading === 'MODEL') {
|
||||
parsedRows.push(['—', '']);
|
||||
parsedRows.push(['section', heading]);
|
||||
sections += 1;
|
||||
}
|
||||
}
|
||||
if (sections !== 3) throw new Error(`the operator report must separate the app, queue, and model boundaries, not ${sections} of them`);
|
||||
const analyzeRow = parsedRows.find(([key]) => /^analyze requests$/i.test(key));
|
||||
if (!analyzeRow || Number(analyzeRow[1]) < 1) {
|
||||
throw new Error(`the report must count this recording's own requests, not a zero: ${reportRun.stdout}`);
|
||||
}
|
||||
const metricsCard = await evidenceCard({
|
||||
id: 'operator-health-report',
|
||||
title: 'OPERATOR HEALTH REPORT',
|
||||
subtitle: 'command line, not app UI — this recording’s own server, demo token',
|
||||
rows: [
|
||||
['command', 'node scripts/model_metrics_report.mjs'],
|
||||
['target', 'http://127.0.0.1:4173 · loopback only'],
|
||||
['—', ''],
|
||||
...parsedRows,
|
||||
['—', ''],
|
||||
['wrong token', 'refused · exit 1 · no counters returned'],
|
||||
],
|
||||
font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
|
||||
keyWidth: 74,
|
||||
gap: '1px',
|
||||
rowGap: '5px',
|
||||
});
|
||||
await sleep(420);
|
||||
const metricsCardProof = await evidenceCardProof('operator-health-report');
|
||||
if (!metricsCardProof.ok) throw new Error(`Operator report card is not fully readable on screen: ${JSON.stringify(metricsCardProof)}`);
|
||||
await caption('The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server', 3800);
|
||||
await caption('Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered', 3600);
|
||||
await caption('The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity', 4000);
|
||||
await page.evaluate(() => document.querySelector('#operator-health-report')?.remove());
|
||||
|
||||
await indicate('#delete-all');
|
||||
await tap('#delete-all', 650);
|
||||
await page.getByText(/Local ledger and photos deleted/).waitFor();
|
||||
|
||||
Reference in new issue
Block a user