feat: ground Timmy chat answers in cited ledger records (#37)
Quality gates / quality (pull_request) Failing after 2m50s
Quality gates / quality (pull_request) Failing after 2m50s
Chat now has to show its work. Every answer that states a count, a date, or a visible form cites the confirmed records behind it, and following a citation opens exactly that log. Questions the journal cannot answer (food safety, medicine, what a symptom means) get boundary copy with no borrowed evidence, and a sparse window refuses to describe a trend instead of naming one. The Hermes path is held to the same rule: the server gives each ledger record a positional ref, the prompt requires inline [entry:L<n>] citations, and the browser resolves them against the ledger it sent. A ref that matches no record is stripped and the answer is labelled unverified rather than rendered as a source, so an invented citation cannot look like evidence. The release recording now exercises the grounded local answer, its citation sheet, the refusal, and a Hermes reply whose refs resolve, each asserted in the browser with fictional logs.
This commit is contained in:
1 parent
3b267c0fff
commit
b838beb548
17 files changed
+556
-18
No files matched your search
@@ -45,6 +45,9 @@ DEMO_REQUIRED_CAPTIONS = (
|
||||
"Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
|
||||
"Every pattern line names the confirmed logs behind it, and following one shows exactly those records",
|
||||
"Two logs is not a pattern: the summary refuses to describe a trend it cannot support",
|
||||
"The answer states the count and cites the four confirmed logs it counted",
|
||||
"Following a citation opens exactly the confirmed log behind that answer",
|
||||
"Ask something the journal cannot answer and Timmy refuses instead of reassuring",
|
||||
)
|
||||
|
||||
|
||||
@@ -144,6 +147,7 @@ def main() -> int:
|
||||
run(["npm", "run", "test:vision-refusal"], tree)
|
||||
run(["npm", "run", "test:worker-readiness"], tree)
|
||||
run(["npm", "run", "test:pattern-summary"], tree)
|
||||
run(["npm", "run", "test:grounded-chat"], tree)
|
||||
demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
|
||||
caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
|
||||
demo_env = dict(server_env)
|
||||
@@ -270,7 +274,7 @@ def main() -> int:
|
||||
"codec": "h264",
|
||||
"pixel_format": "yuv420p",
|
||||
"recorded_from_working_app": True,
|
||||
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, fictional multi-day sample logs driving the explainable pattern summary and its citation sheet, a sparse fictional window showing the insufficient-data copy, and deterministic Hermes chat response",
|
||||
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, fictional multi-day sample logs driving the explainable pattern summary and its citation sheet, a sparse fictional window showing the insufficient-data copy, a grounded local chat answer citing each counted log and opening the cited record, a refusal that borrows no journal evidence, and a deterministic Hermes chat reply whose positional refs render as sources",
|
||||
},
|
||||
"gates": {
|
||||
"unit_security": "passed",
|
||||
@@ -284,6 +288,7 @@ def main() -> int:
|
||||
"worker_container_receipts": "passed",
|
||||
"worker_post_load_readiness": "passed",
|
||||
"longitudinal_pattern_summary": "passed",
|
||||
"grounded_chat_citations": "passed",
|
||||
"dependency_audit_high": "passed",
|
||||
"syntax": "passed",
|
||||
"secret_scan": "passed",
|
||||
|
||||
@@ -82,18 +82,29 @@ await page.route('**/api/analyze', route => {
|
||||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||||
});
|
||||
let hermesCalls = 0;
|
||||
// Issue #37 turns the agent on and off on camera: local Timmy mode is a real app state, and the
|
||||
// grounded local answer has to be shown as its own surface rather than implied.
|
||||
let agentConnected = true;
|
||||
await page.route('**/api/agent/status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }),
|
||||
body: JSON.stringify({
|
||||
enabled: true,
|
||||
// Local Timmy mode is the no-backend state the app really uses: nothing configured, nothing sent.
|
||||
configured: agentConnected,
|
||||
authenticated: agentConnected,
|
||||
mode: agentConnected ? 'hermes-agent' : 'local-fallback',
|
||||
}),
|
||||
}));
|
||||
let hermesCalls = 0;
|
||||
await page.route('**/api/agent/chat', route => {
|
||||
hermesCalls += 1;
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ connected: true, reply: 'Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
|
||||
// The refs are the positional handles the server gives the agent; the browser resolves them
|
||||
// against the same ledger it sent, so this reply is a real citation contract, not a prop.
|
||||
body: JSON.stringify({ connected: true, reply: 'You have 4 confirmed logs in the last 28 days [entry:L1] [entry:L2] [entry:L3] [entry:L4]. Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -373,12 +384,55 @@ await caption('Export writes a plain JSON file you hold — the model records it
|
||||
await indicate('#delete-all');
|
||||
await caption('Delete all local data clears the ledger in this browser; there is no server copy of it to keep', 2600);
|
||||
await tap('[data-view="home"]', 500);
|
||||
// Issue #37 — a chat answer about the journal must point at the confirmed records behind it. The
|
||||
// scenes below run against the app's own local ledger contract with fictional logs, and every claim
|
||||
// is asserted in the browser before the caption is shown.
|
||||
agentConnected = false;
|
||||
await seedSyntheticLedger([
|
||||
sampleDay('demo-chat-1', 2, 4),
|
||||
sampleDay('demo-chat-2', 5, 3),
|
||||
sampleDay('demo-chat-3', 9, 4),
|
||||
sampleDay('demo-chat-4', 12, 4),
|
||||
]);
|
||||
await tap('[data-view="timmy"]', 550);
|
||||
await page.getByText(/Local Timmy mode/i).waitFor();
|
||||
await caption('Fictional logs are seeded into this browser only — the chat answers from them, not from model memory', 2600);
|
||||
await page.locator('#chat-message').fill('How many times have I logged?');
|
||||
await tap('#send-chat', 500);
|
||||
const groundedAnswer = page.locator('.bubble.timmy', { hasText: '4 confirmed logs' }).last();
|
||||
await groundedAnswer.locator('[data-chat-citation]').first().waitFor();
|
||||
if (await groundedAnswer.locator('[data-chat-citation]').count() !== 4) {
|
||||
throw new Error('A grounded chat answer must cite every record it counted');
|
||||
}
|
||||
await caption('The answer states the count and cites the four confirmed logs it counted', 2600);
|
||||
await indicate('[data-chat-citations]');
|
||||
await tap('[data-chat-citation]', 700);
|
||||
await page.locator('[data-chat-record]').waitFor();
|
||||
if (await page.locator('[data-chat-record]').count() !== 1) {
|
||||
throw new Error('A chat citation must open exactly the one record it names');
|
||||
}
|
||||
await caption('Following a citation opens exactly the confirmed log behind that answer', 2600);
|
||||
await indicate('[data-chat-record-sheet]');
|
||||
await tap('#chat-record-close', 450);
|
||||
await page.locator('#chat-message').fill('Is it safe to eat tacos?');
|
||||
await tap('#send-chat', 500);
|
||||
await page.getByText(/cannot clear a food/i).waitFor();
|
||||
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 0) {
|
||||
throw new Error('A refusal must not borrow journal citations');
|
||||
}
|
||||
await caption('Ask something the journal cannot answer and Timmy refuses instead of reassuring', 2800);
|
||||
agentConnected = true;
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
await tap('[data-view="timmy"]', 550);
|
||||
await page.getByText(/Hermes Agent connected/i).waitFor();
|
||||
await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400);
|
||||
await page.locator('#chat-message').fill('What pattern do you see?');
|
||||
await tap('#send-chat', 500);
|
||||
await page.getByText(/mostly Type 4/i).waitFor();
|
||||
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 4) {
|
||||
throw new Error('A Hermes answer must render the records it cites as sources');
|
||||
}
|
||||
await caption('Hermes answers carry the same ledger citations, rendered as sources in the transcript', 2400);
|
||||
await caption('Hermes keeps credentials, tools, and session continuity server-side. Photos stay out of chat.', 1600);
|
||||
await page.locator('#chat-message').fill('I barfed');
|
||||
await tap('#send-chat', 450);
|
||||
|
||||
Reference in new issue
Block a user