feat: ground Timmy chat answers in cited ledger records (#37)
Quality gates / quality (pull_request) Failing after 2m50s

Chat now has to show its work. Every answer that states a count, a date, or a
visible form cites the confirmed records behind it, and following a citation
opens exactly that log. Questions the journal cannot answer (food safety,
medicine, what a symptom means) get boundary copy with no borrowed evidence,
and a sparse window refuses to describe a trend instead of naming one.

The Hermes path is held to the same rule: the server gives each ledger record a
positional ref, the prompt requires inline [entry:L<n>] citations, and the
browser resolves them against the ledger it sent. A ref that matches no record
is stripped and the answer is labelled unverified rather than rendered as a
source, so an invented citation cannot look like evidence.

The release recording now exercises the grounded local answer, its citation
sheet, the refusal, and a Hermes reply whose refs resolve, each asserted in the
browser with fictional logs.
This commit is contained in:
timmy committed 2026-10-18 08:17:40 +00:00
1 parent 3b267c0fff
commit b838beb548
17 files changed
+556 -18

No files matched your search

+6 -1
View File
@@ -45,6 +45,9 @@ DEMO_REQUIRED_CAPTIONS = (
"Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
"Every pattern line names the confirmed logs behind it, and following one shows exactly those records",
"Two logs is not a pattern: the summary refuses to describe a trend it cannot support",
"The answer states the count and cites the four confirmed logs it counted",
"Following a citation opens exactly the confirmed log behind that answer",
"Ask something the journal cannot answer and Timmy refuses instead of reassuring",
)
@@ -144,6 +147,7 @@ def main() -> int:
run(["npm", "run", "test:vision-refusal"], tree)
run(["npm", "run", "test:worker-readiness"], tree)
run(["npm", "run", "test:pattern-summary"], tree)
run(["npm", "run", "test:grounded-chat"], tree)
demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
demo_env = dict(server_env)
@@ -270,7 +274,7 @@ def main() -> int:
"codec": "h264",
"pixel_format": "yuv420p",
"recorded_from_working_app": True,
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, fictional multi-day sample logs driving the explainable pattern summary and its citation sheet, a sparse fictional window showing the insufficient-data copy, and deterministic Hermes chat response",
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, fictional multi-day sample logs driving the explainable pattern summary and its citation sheet, a sparse fictional window showing the insufficient-data copy, a grounded local chat answer citing each counted log and opening the cited record, a refusal that borrows no journal evidence, and a deterministic Hermes chat reply whose positional refs render as sources",
},
"gates": {
"unit_security": "passed",
@@ -284,6 +288,7 @@ def main() -> int:
"worker_container_receipts": "passed",
"worker_post_load_readiness": "passed",
"longitudinal_pattern_summary": "passed",
"grounded_chat_citations": "passed",
"dependency_audit_high": "passed",
"syntax": "passed",
"secret_scan": "passed",
+57 -3
View File
@@ -82,18 +82,29 @@ await page.route('**/api/analyze', route => {
const response = analyzeOverride || { status: 200, body: suggestionBody };
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
});
let hermesCalls = 0;
// Issue #37 turns the agent on and off on camera: local Timmy mode is a real app state, and the
// grounded local answer has to be shown as its own surface rather than implied.
let agentConnected = true;
await page.route('**/api/agent/status', route => route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }),
body: JSON.stringify({
enabled: true,
// Local Timmy mode is the no-backend state the app really uses: nothing configured, nothing sent.
configured: agentConnected,
authenticated: agentConnected,
mode: agentConnected ? 'hermes-agent' : 'local-fallback',
}),
}));
let hermesCalls = 0;
await page.route('**/api/agent/chat', route => {
hermesCalls += 1;
return route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ connected: true, reply: 'Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
// The refs are the positional handles the server gives the agent; the browser resolves them
// against the same ledger it sent, so this reply is a real citation contract, not a prop.
body: JSON.stringify({ connected: true, reply: 'You have 4 confirmed logs in the last 28 days [entry:L1] [entry:L2] [entry:L3] [entry:L4]. Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
});
});
@@ -373,12 +384,55 @@ await caption('Export writes a plain JSON file you hold — the model records it
await indicate('#delete-all');
await caption('Delete all local data clears the ledger in this browser; there is no server copy of it to keep', 2600);
await tap('[data-view="home"]', 500);
// Issue #37 — a chat answer about the journal must point at the confirmed records behind it. The
// scenes below run against the app's own local ledger contract with fictional logs, and every claim
// is asserted in the browser before the caption is shown.
agentConnected = false;
await seedSyntheticLedger([
sampleDay('demo-chat-1', 2, 4),
sampleDay('demo-chat-2', 5, 3),
sampleDay('demo-chat-3', 9, 4),
sampleDay('demo-chat-4', 12, 4),
]);
await tap('[data-view="timmy"]', 550);
await page.getByText(/Local Timmy mode/i).waitFor();
await caption('Fictional logs are seeded into this browser only — the chat answers from them, not from model memory', 2600);
await page.locator('#chat-message').fill('How many times have I logged?');
await tap('#send-chat', 500);
const groundedAnswer = page.locator('.bubble.timmy', { hasText: '4 confirmed logs' }).last();
await groundedAnswer.locator('[data-chat-citation]').first().waitFor();
if (await groundedAnswer.locator('[data-chat-citation]').count() !== 4) {
throw new Error('A grounded chat answer must cite every record it counted');
}
await caption('The answer states the count and cites the four confirmed logs it counted', 2600);
await indicate('[data-chat-citations]');
await tap('[data-chat-citation]', 700);
await page.locator('[data-chat-record]').waitFor();
if (await page.locator('[data-chat-record]').count() !== 1) {
throw new Error('A chat citation must open exactly the one record it names');
}
await caption('Following a citation opens exactly the confirmed log behind that answer', 2600);
await indicate('[data-chat-record-sheet]');
await tap('#chat-record-close', 450);
await page.locator('#chat-message').fill('Is it safe to eat tacos?');
await tap('#send-chat', 500);
await page.getByText(/cannot clear a food/i).waitFor();
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 0) {
throw new Error('A refusal must not borrow journal citations');
}
await caption('Ask something the journal cannot answer and Timmy refuses instead of reassuring', 2800);
agentConnected = true;
await page.reload({ waitUntil: 'networkidle' });
await tap('[data-view="timmy"]', 550);
await page.getByText(/Hermes Agent connected/i).waitFor();
await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400);
await page.locator('#chat-message').fill('What pattern do you see?');
await tap('#send-chat', 500);
await page.getByText(/mostly Type 4/i).waitFor();
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 4) {
throw new Error('A Hermes answer must render the records it cites as sources');
}
await caption('Hermes answers carry the same ledger citations, rendered as sources in the transcript', 2400);
await caption('Hermes keeps credentials, tools, and session continuity server-side. Photos stay out of chat.', 1600);
await page.locator('#chat-message').fill('I barfed');
await tap('#send-chat', 450);