timmy
b838beb548
Quality gates / quality (pull_request) Failing after 2m50s
Chat now has to show its work. Every answer that states a count, a date, or a visible form cites the confirmed records behind it, and following a citation opens exactly that log. Questions the journal cannot answer (food safety, medicine, what a symptom means) get boundary copy with no borrowed evidence, and a sparse window refuses to describe a trend instead of naming one. The Hermes path is held to the same rule: the server gives each ledger record a positional ref, the prompt requires inline [entry:L<n>] citations, and the browser resolves them against the ledger it sent. A ref that matches no record is stripped and the answer is labelled unverified rather than rendered as a source, so an invented citation cannot look like evidence. The release recording now exercises the grounded local answer, its citation sheet, the refusal, and a Hermes reply whose refs resolve, each asserted in the browser with fictional logs.
115 lines
5.8 KiB
JavaScript
115 lines
5.8 KiB
JavaScript
import { chromium } from 'playwright';
|
|
import assert from 'node:assert/strict';
|
|
import { mkdir } from 'node:fs/promises';
|
|
|
|
// Issue #37: a chat answer about the journal must point at the confirmed records behind it, refuse
|
|
// when the journal cannot answer, and never render a citation that is not a real record. Every log
|
|
// here is fictional sample data written into the app's own local ledger contract.
|
|
const DAY = 86400000;
|
|
const sample = (id, daysAgo, bristolType) => ({
|
|
id,
|
|
occurredAt: new Date(Date.now() - daysAgo * DAY).toISOString(),
|
|
bristolType,
|
|
color: 'brown',
|
|
urgency: 1,
|
|
discomfort: 1,
|
|
note: '',
|
|
photoDataUrl: '',
|
|
symptoms: {},
|
|
});
|
|
|
|
await mkdir('artifacts', { recursive: true });
|
|
const browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' });
|
|
const page = await context.newPage();
|
|
const errors = [];
|
|
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
|
page.on('pageerror', error => errors.push(error.message));
|
|
|
|
const seed = async entries => page.evaluate(list => {
|
|
const base = document.querySelector('meta[name="timmy-base-path"]')?.content || '/';
|
|
localStorage.setItem(`timmy:${base}:ledger-v1`, JSON.stringify(list));
|
|
}, entries);
|
|
|
|
const ask = async question => {
|
|
await page.locator('#chat-message').fill(question);
|
|
await page.locator('#send-chat').click();
|
|
await page.waitForFunction(() => !document.querySelector('.bubble.timmy.thinking'));
|
|
};
|
|
|
|
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
|
await page.evaluate(() => localStorage.clear());
|
|
|
|
// 1. A journal question answers with the count and cites the records it counted.
|
|
await seed([sample('chat-a', 2, 4), sample('chat-b', 5, 3), sample('chat-c', 9, 6), sample('chat-d', 12, 4)]);
|
|
await page.reload({ waitUntil: 'networkidle' });
|
|
await page.locator('[data-view="timmy"]').last().click();
|
|
await page.getByText(/Local Timmy mode|Hermes Agent connected/i).waitFor();
|
|
await ask('How many times have I logged?');
|
|
|
|
const answer = page.locator('.bubble.timmy').last();
|
|
assert.match(await answer.innerText(), /4 confirmed logs/);
|
|
const chips = answer.locator('[data-chat-citation]');
|
|
assert.equal(await chips.count(), 4, 'every counted log is offered as a citation');
|
|
await page.screenshot({ path: 'artifacts/grounded-chat-citations-mobile.png' });
|
|
|
|
// 2. Following a citation shows exactly that confirmed record.
|
|
await chips.first().click();
|
|
const sheet = page.locator('[data-chat-record-sheet]');
|
|
await sheet.waitFor();
|
|
assert.equal(await page.locator('[data-chat-record]').count(), 1, 'a citation opens the single record it names');
|
|
assert.match(await sheet.innerText(), /Type \d/);
|
|
assert.match(await sheet.innerText(), /confirmed log/i);
|
|
await page.screenshot({ path: 'artifacts/grounded-chat-citation-record-mobile.png' });
|
|
await page.locator('#chat-record-close').click();
|
|
assert.equal(await page.locator('[data-chat-record-sheet]').count(), 0, 'the record sheet closes cleanly');
|
|
|
|
// 3. A question the journal cannot answer gets boundary copy and no citations.
|
|
await ask('Is it safe to eat tacos?');
|
|
const refusal = page.locator('.bubble.timmy').last();
|
|
assert.match(await refusal.innerText(), /cannot clear a food/i);
|
|
assert.equal(await refusal.locator('[data-chat-citation]').count(), 0, 'a refusal must not borrow journal evidence');
|
|
await page.screenshot({ path: 'artifacts/grounded-chat-refusal-mobile.png' });
|
|
|
|
// 4. An urgent disclosure still routes to the deterministic escalation, with no citations attached.
|
|
await ask('I have blood in my stool');
|
|
const escalation = page.locator('.bubble.timmy').last();
|
|
assert.match(await escalation.innerText(), /Pause and get medical help/i);
|
|
assert.equal(await escalation.locator('[data-chat-citation]').count(), 0);
|
|
|
|
// 5. A sparse journal refuses to describe a pattern.
|
|
await seed([sample('chat-sparse-a', 1, 4), sample('chat-sparse-b', 4, 6)]);
|
|
await page.reload({ waitUntil: 'networkidle' });
|
|
await page.locator('[data-view="timmy"]').last().click();
|
|
await ask('What pattern do you see?');
|
|
const sparse = page.locator('.bubble.timmy').last();
|
|
assert.match(await sparse.innerText(), /not enough confirmed logs/i);
|
|
assert.equal(await sparse.locator('[data-chat-citation]').count(), 0);
|
|
|
|
// 6. An agent answer that cites a record which does not exist is shown as unverified, not as a source.
|
|
await page.route('**/api/agent/status', route => route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }),
|
|
}));
|
|
await page.route('**/api/agent/chat', route => route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ connected: true, reply: 'You logged 9 times this month [entry:L99].' }),
|
|
}));
|
|
await seed([sample('chat-a', 2, 4), sample('chat-b', 5, 3), sample('chat-c', 9, 6), sample('chat-d', 12, 4)]);
|
|
await page.reload({ waitUntil: 'networkidle' });
|
|
await page.locator('[data-view="timmy"]').last().click();
|
|
await page.getByText(/Hermes Agent connected/i).waitFor();
|
|
await ask('What pattern do you see?');
|
|
const unverified = page.locator('.bubble.timmy').last();
|
|
assert.match(await page.locator('[data-chat-unverified]').innerText(), /not verified against your journal/i);
|
|
assert.equal(await unverified.locator('[data-chat-citation]').count(), 0, 'an invented ref never becomes a citation');
|
|
assert.equal(await page.locator('#chat').innerText().then(text => text.includes('[entry:')), false, 'raw markers never reach the user');
|
|
await page.screenshot({ path: 'artifacts/grounded-chat-unverified-mobile.png' });
|
|
|
|
assert.deepEqual(errors, []);
|
|
await context.close();
|
|
await browser.close();
|
|
console.log('PASS grounded chat cites confirmed records, refuses uncitable questions, and flags unverified answers');
|