feat: graceful shutdown, disconnect-driven cancellation, and queue acceptance coverage
All checks were successful
Quality gates / quality (pull_request) Successful in 1m58s
All checks were successful
Quality gates / quality (pull_request) Successful in 1m58s
- server.mjs tracks in-flight chats: browser disconnects abort the queued turn (releasing its slot), and SIGTERM/SIGINT cancels every in-flight turn so no Hermes subprocess is orphaned. - vision-service forwards a per-request AbortSignal combined with the provider deadline via AbortSignal.any. - acceptance suites cover overload fallback, urgent bypass under saturation, disconnect cleanup, recovery, and shutdown orphan checks. - zero-call invariant tests prove urgent text never reaches Hermes or the vision provider under load.
This commit is contained in:
parent
5dcaaae4d6
commit
9118b69057
|
|
@ -4,12 +4,14 @@
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js",
|
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-queue.test.js tests/agent-turn-timeout.test.js tests/inference-queue.test.js tests/inference-queue-cancellation.test.js tests/inference-zero-call.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js",
|
||||||
"test:ui": "node tests/ui.acceptance.mjs",
|
"test:ui": "node tests/ui.acceptance.mjs",
|
||||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||||
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
||||||
|
"test:queue-acceptance": "node tests/inference-queue.acceptance.mjs",
|
||||||
|
"test:shutdown-acceptance": "node tests/server-shutdown.acceptance.mjs",
|
||||||
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
||||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py",
|
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/inference-queue.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/inference-queue.acceptance.mjs && node --check tests/server-shutdown.acceptance.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py",
|
||||||
"check:diff": "bash scripts/check_diff.sh",
|
"check:diff": "bash scripts/check_diff.sh",
|
||||||
"start": "node server.mjs"
|
"start": "node server.mjs"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
30
server.mjs
30
server.mjs
|
|
@ -39,6 +39,34 @@ function rejectCrossSite(req){const site=String(req.headers['sec-fetch-site']||'
|
||||||
function agentCookie(token){const secure=agentConfig.publicOrigin.startsWith('https://')?'; Secure':'';return `timmy_agent=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=${appRoot}; Max-Age=86400${secure}`}
|
function agentCookie(token){const secure=agentConfig.publicOrigin.startsWith('https://')?'; Secure':'';return `timmy_agent=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=${appRoot}; Max-Age=86400${secure}`}
|
||||||
function sendAgentError(res,error){const status=error instanceof AgentGatewayError?error.status:503;const message=error instanceof AgentGatewayError?error.message:'Hermes is temporarily unavailable. Your local journal still works.';return sendJson(res,status,{error:message})}
|
function sendAgentError(res,error){const status=error instanceof AgentGatewayError?error.status:503;const message=error instanceof AgentGatewayError?error.message:'Hermes is temporarily unavailable. Your local journal still works.';return sendJson(res,status,{error:message})}
|
||||||
|
|
||||||
|
// Graceful shutdown: SIGTERM must cancel in-flight Hermes turns (killing their
|
||||||
|
// child processes) instead of abandoning them as orphan subprocesses.
|
||||||
|
const inFlightChats = new Set();
|
||||||
|
let shuttingDown = false;
|
||||||
|
function trackChat(res, controller) {
|
||||||
|
if (shuttingDown) controller.abort(new Error('Timmy is shutting down. Your journal still works — try again shortly.'));
|
||||||
|
const entry = { res, controller };
|
||||||
|
inFlightChats.add(entry);
|
||||||
|
// A vanished client must release its queue slot and stop any in-flight
|
||||||
|
// Hermes turn instead of consuming inference capacity silently.
|
||||||
|
res.on('close', () => {
|
||||||
|
inFlightChats.delete(entry);
|
||||||
|
if (!res.writableEnded) {
|
||||||
|
try { controller.abort(new Error('The browser closed the request. Your journal still works.')); } catch { /* already aborted */ }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
for (const signalName of ['SIGTERM', 'SIGINT']) {
|
||||||
|
process.on(signalName, () => {
|
||||||
|
shuttingDown = true;
|
||||||
|
for (const { controller } of inFlightChats) {
|
||||||
|
try { controller.abort(new Error('Server is shutting down.')); } catch { /* already aborted */ }
|
||||||
|
}
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
http.createServer(async(req,res)=>{
|
http.createServer(async(req,res)=>{
|
||||||
try{
|
try{
|
||||||
const url=new URL(req.url,'http://localhost');
|
const url=new URL(req.url,'http://localhost');
|
||||||
|
|
@ -62,7 +90,7 @@ http.createServer(async(req,res)=>{
|
||||||
try{rejectCrossSite(req);const payload=await readJson(req,4096);const result=await agentService.unlock({origin:requestOrigin(req),accessCode:String(payload?.accessCode||'')});res.setHeader('set-cookie',agentCookie(result.cookieToken));return sendJson(res,200,result.public)}catch(error){return sendAgentError(res,error)}
|
try{rejectCrossSite(req);const payload=await readJson(req,4096);const result=await agentService.unlock({origin:requestOrigin(req),accessCode:String(payload?.accessCode||'')});res.setHeader('set-cookie',agentCookie(result.cookieToken));return sendJson(res,200,result.public)}catch(error){return sendAgentError(res,error)}
|
||||||
}
|
}
|
||||||
if(appPath==='/api/agent/chat'&&req.method==='POST'){
|
if(appPath==='/api/agent/chat'&&req.method==='POST'){
|
||||||
try{rejectCrossSite(req);const payload=await readJson(req,128*1024);return sendJson(res,200,await agentService.chat({origin:requestOrigin(req),cookieToken:cookie(req,'timmy_agent'),payload}))}catch(error){return sendAgentError(res,error)}
|
try{rejectCrossSite(req);const payload=await readJson(req,128*1024);const chatAbort=new AbortController();trackChat(res,chatAbort);return sendJson(res,200,await agentService.chat({origin:requestOrigin(req),cookieToken:cookie(req,'timmy_agent'),payload,signal:chatAbort.signal}))}catch(error){return sendAgentError(res,error)}
|
||||||
}
|
}
|
||||||
if(appPath.startsWith('/api/'))return sendJson(res,404,{error:'Not found'});
|
if(appPath.startsWith('/api/'))return sendJson(res,404,{error:'Not found'});
|
||||||
if(req.method!=='GET'&&req.method!=='HEAD'){res.writeHead(405,{'allow':'GET, HEAD'});return res.end()}
|
if(req.method!=='GET'&&req.method!=='HEAD'){res.writeHead(405,{'allow':'GET, HEAD'});return res.end()}
|
||||||
|
|
|
||||||
|
|
@ -7,18 +7,22 @@ function providerEndpoint(baseUrl) {
|
||||||
return `${url.toString().replace(/\/$/, '')}/chat/completions`;
|
return `${url.toString().replace(/\/$/, '')}/chat/completions`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function analyzePhoto({ payload, fetchImpl = fetch, config }) {
|
export async function analyzePhoto({ payload, fetchImpl = fetch, config, signal }) {
|
||||||
const photo = validatePhotoPayload(payload);
|
const photo = validatePhotoPayload(payload);
|
||||||
if (!config?.model) throw new Error('AI analysis is not configured.');
|
if (!config?.model) throw new Error('AI analysis is not configured.');
|
||||||
const endpoint = providerEndpoint(config.baseUrl);
|
const endpoint = providerEndpoint(config.baseUrl);
|
||||||
|
// The provider call honours both its own deadline and a per-request
|
||||||
|
// cancellation so abandoned clients stop consuming inference capacity.
|
||||||
|
const timeoutSignal = AbortSignal.timeout(config.requestTimeoutMs || 60_000);
|
||||||
|
const combinedSignal = signal ? AbortSignal.any([timeoutSignal, signal]) : timeoutSignal;
|
||||||
const response = await fetchImpl(endpoint, {
|
const response = await fetchImpl(endpoint, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'content-type': 'application/json',
|
'content-type': 'application/json',
|
||||||
authorization: `Bearer ${config.apiKey || 'local-proxy'}`,
|
authorization: 'Bearer ' + String(config.apiKey || 'local-proxy'),
|
||||||
},
|
},
|
||||||
body: JSON.stringify(buildVisionRequest({ imageDataUrl: photo.imageDataUrl, model: config.model })),
|
body: JSON.stringify(buildVisionRequest({ imageDataUrl: photo.imageDataUrl, model: config.model })),
|
||||||
signal: AbortSignal.timeout(config.requestTimeoutMs || 60_000),
|
signal: combinedSignal,
|
||||||
}).catch(() => { throw new Error('AI analysis is temporarily unavailable. Continue manually.'); });
|
}).catch(() => { throw new Error('AI analysis is temporarily unavailable. Continue manually.'); });
|
||||||
if (!response.ok) throw new Error('AI analysis is temporarily unavailable. Continue manually.');
|
if (!response.ok) throw new Error('AI analysis is temporarily unavailable. Continue manually.');
|
||||||
let data;
|
let data;
|
||||||
|
|
|
||||||
20
tests/fixtures/slow-hermes.mjs
vendored
Executable file
20
tests/fixtures/slow-hermes.mjs
vendored
Executable file
|
|
@ -0,0 +1,20 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
// Slow Hermes fixture for bounded-queue acceptance tests. Normal prompts
|
||||||
|
// answer immediately; prompts containing HOLD block until a release file
|
||||||
|
// appears inside the agent workdir (the value after `--in`), so tests can
|
||||||
|
// saturate the queue deterministically without touching shared /tmp state.
|
||||||
|
import { existsSync } from 'node:fs';
|
||||||
|
import { appendFileSync } from 'node:fs';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const promptIndex = args.indexOf('-q');
|
||||||
|
const prompt = promptIndex >= 0 ? args[promptIndex + 1] : '';
|
||||||
|
const workdirIndex = args.indexOf('--in');
|
||||||
|
const workdir = workdirIndex >= 0 ? args[workdirIndex + 1] : '/tmp';
|
||||||
|
appendFileSync(`${workdir}/fixture-pids.log`, `${process.pid}\n`);
|
||||||
|
process.stdout.write('session_id: slow_fixture_2026\n');
|
||||||
|
if (/HOLD/.test(prompt)) {
|
||||||
|
const releaseFile = `${workdir}/hermes-release`;
|
||||||
|
while (!existsSync(releaseFile)) await new Promise(resolve => setTimeout(resolve, 20));
|
||||||
|
}
|
||||||
|
process.stdout.write('Hermes fixture answered the bounded request.\n');
|
||||||
136
tests/inference-queue.acceptance.mjs
Normal file
136
tests/inference-queue.acceptance.mjs
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { chmod, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { constants as fsConstants } from 'node:fs';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const root = fileURLToPath(new URL('..', import.meta.url));
|
||||||
|
const fixture = fileURLToPath(new URL('./fixtures/slow-hermes.mjs', import.meta.url));
|
||||||
|
const origin = 'http://127.0.0.1:4187';
|
||||||
|
const accessCode = 'queue-acceptance-code-2026';
|
||||||
|
|
||||||
|
async function waitReady(child) {
|
||||||
|
const deadline = Date.now() + 10_000;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
if (child.exitCode !== null) throw new Error(`server exited ${child.exitCode}`);
|
||||||
|
try { const response = await fetch(`${origin}/api/healthz`); if (response.ok) return; } catch {}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
throw new Error('server did not become ready');
|
||||||
|
}
|
||||||
|
|
||||||
|
function post(path, body, { cookie = '', signal } = {}) {
|
||||||
|
return fetch(`${origin}${path}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
origin,
|
||||||
|
'sec-fetch-site': 'same-origin',
|
||||||
|
...(cookie ? { cookie } : {}),
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
signal,
|
||||||
|
}).then(
|
||||||
|
async response => ({ status: response.status, body: await response.json().catch(() => ({})) }),
|
||||||
|
error => ({ failed: true, reason: String(error?.name || error) }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test('bounded queue end-to-end: disconnect cleanup, overload fallback, urgent bypass, recovery', async t => {
|
||||||
|
const workdir = await mkdtemp(join(tmpdir(), 'timmy-queue-accept-'));
|
||||||
|
const releaseFile = join(workdir, 'hermes-release');
|
||||||
|
const pidLog = join(workdir, 'fixture-pids.log');
|
||||||
|
await chmod(fixture, fsConstants.S_IRWXU);
|
||||||
|
const child = spawn(process.execPath, ['server.mjs'], {
|
||||||
|
cwd: root,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
PORT: '4187',
|
||||||
|
TIMMY_AGENT_ENABLED: 'true',
|
||||||
|
TIMMY_AGENT_ACCESS_TOKEN: accessCode,
|
||||||
|
TIMMY_PUBLIC_ORIGIN: origin,
|
||||||
|
TIMMY_AGENT_WORKDIR: workdir,
|
||||||
|
TIMMY_HERMES_COMMAND: fixture,
|
||||||
|
TIMMY_AGENT_MAX_CONCURRENT_TURNS: '1',
|
||||||
|
TIMMY_AGENT_MAX_QUEUE_DEPTH: '2',
|
||||||
|
},
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
});
|
||||||
|
let serverLogs = '';
|
||||||
|
child.stderr.on('data', chunk => { serverLogs += String(chunk); });
|
||||||
|
t.after(async () => {
|
||||||
|
child.kill('SIGTERM');
|
||||||
|
await rm(workdir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
await rm(releaseFile, { force: true }).catch(() => {});
|
||||||
|
});
|
||||||
|
await waitReady(child);
|
||||||
|
|
||||||
|
// Authenticate once and keep the cookie for the whole scenario.
|
||||||
|
const rawResponse = await fetch(`${origin}/api/agent/unlock`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json', origin, 'sec-fetch-site': 'same-origin' },
|
||||||
|
body: JSON.stringify({ accessCode }),
|
||||||
|
});
|
||||||
|
assert.equal(rawResponse.status, 200);
|
||||||
|
const browserCookie = rawResponse.headers.get('set-cookie').split(';', 1)[0];
|
||||||
|
|
||||||
|
// Fill the slot: this HOLD runs until the fixture release file appears.
|
||||||
|
const heldOne = post('/api/agent/chat', { message: 'HOLD one', ledger: [] }, { cookie: browserCookie });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 400));
|
||||||
|
|
||||||
|
// 1. Disconnect cleanup: this request waits in the queue. When its client
|
||||||
|
// vanishes, Timmy must drop it and free the queued place.
|
||||||
|
const disconnectController = new AbortController();
|
||||||
|
const doomed = post('/api/agent/chat', { message: 'HOLD doomed', ledger: [] }, { cookie: browserCookie, signal: disconnectController.signal });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 400));
|
||||||
|
disconnectController.abort();
|
||||||
|
const doomedOutcome = await doomed;
|
||||||
|
assert.equal(doomedOutcome.failed, true, 'abandoned client saw its request cancelled');
|
||||||
|
|
||||||
|
// If the freed place was reclaimed, both follow-up HOLDs are admitted
|
||||||
|
// (slot + queue of two minus one freed place). A sanitized 503 here would
|
||||||
|
// mean the cancelled client did not release capacity.
|
||||||
|
const heldTwo = post('/api/agent/chat', { message: 'HOLD two', ledger: [] }, { cookie: browserCookie });
|
||||||
|
const heldThree = post('/api/agent/chat', { message: 'HOLD three', ledger: [] }, { cookie: browserCookie });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 500));
|
||||||
|
|
||||||
|
// 2. Overload: everything is saturated now, so a further request must fail
|
||||||
|
// fast with the stable manual-fallback copy and zero provider contact.
|
||||||
|
const overloadResult = await post('/api/agent/chat', { message: 'overload probe', ledger: [] }, { cookie: browserCookie });
|
||||||
|
assert.equal(overloadResult.status, 503);
|
||||||
|
assert.match(overloadResult.body.error, /busy|try again|manually/i);
|
||||||
|
assert.doesNotMatch(overloadResult.body.error, /spawn|hermes|session|token|workdir|fixture/i);
|
||||||
|
|
||||||
|
// 3. Urgent bypass while saturated: deterministic safety answer, zero calls.
|
||||||
|
const urgentResult = await post('/api/agent/chat', { message: 'I have rectal bleeding', ledger: [] }, { cookie: browserCookie });
|
||||||
|
assert.equal(urgentResult.status, 200);
|
||||||
|
assert.equal(urgentResult.body.safetyOverride, true);
|
||||||
|
assert.match(urgentResult.body.reply, /medical help/i);
|
||||||
|
|
||||||
|
// 4. Recovery: releasing the fixture lets every surviving request complete.
|
||||||
|
await writeFile(releaseFile, 'go');
|
||||||
|
for (const [label, promise] of [['one', heldOne], ['two', heldTwo], ['three', heldThree]]) {
|
||||||
|
const settled = await Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise(resolve => setTimeout(() => resolve({ status: 'TEST-TIMEOUT' }), 12_000)),
|
||||||
|
]);
|
||||||
|
assert.equal(settled.status, 200, `${label} completed after recovery`);
|
||||||
|
assert.match(settled.body.reply, /bounded request/);
|
||||||
|
assert.doesNotMatch(JSON.stringify(settled.body), /slow_fixture_2026|session_id/);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. No orphan subprocesses: every fixture PID recorded at spawn is gone.
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 400));
|
||||||
|
const recordedPids = ((await readFile(pidLog, 'utf8').catch(() => '')) || '')
|
||||||
|
.split('\n').map(Number).filter(Boolean);
|
||||||
|
assert.equal(recordedPids.length, 3, `exactly three turns ran (saw ${recordedPids.length})`);
|
||||||
|
for (const pid of [...new Set(recordedPids)]) {
|
||||||
|
let alive = true;
|
||||||
|
try { process.kill(pid, 0); } catch { alive = false; }
|
||||||
|
assert.equal(alive, false, `fixture process ${pid} outlived its turn`);
|
||||||
|
}
|
||||||
|
assert.doesNotMatch(serverLogs, /access-code|secret|token/i);
|
||||||
|
}, { timeout: 40_000 });
|
||||||
108
tests/inference-zero-call.test.js
Normal file
108
tests/inference-zero-call.test.js
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
import { createHermesAgentService, resolveHermesAgentConfig } from '../src/hermes-agent-service.js';
|
||||||
|
import { analyzePhoto } from '../src/vision-service.js';
|
||||||
|
import { OverloadError, createInferenceQueue } from '../src/inference-queue.js';
|
||||||
|
|
||||||
|
const origin = 'http://127.0.0.1:4173';
|
||||||
|
const accessCode = 'test-agent-access-code-2026';
|
||||||
|
const configured = () => resolveHermesAgentConfig({
|
||||||
|
TIMMY_AGENT_ENABLED: 'true',
|
||||||
|
TIMMY_AGENT_ACCESS_TOKEN: accessCode,
|
||||||
|
TIMMY_PUBLIC_ORIGIN: origin,
|
||||||
|
TIMMY_AGENT_WORKDIR: '/tmp/timmy-agent-workspace',
|
||||||
|
});
|
||||||
|
|
||||||
|
function deferred() {
|
||||||
|
let resolve;
|
||||||
|
let reject;
|
||||||
|
const promise = new Promise((res, rej) => { resolve = res; reject = rej; });
|
||||||
|
return { promise, resolve, reject };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('urgent messages intercept before Hermes even when the queue is saturated', async () => {
|
||||||
|
let clock = 0;
|
||||||
|
let hermesCalls = 0;
|
||||||
|
const gate = deferred();
|
||||||
|
const service = createHermesAgentService({
|
||||||
|
config: { ...configured(), maxQueueDepth: 0 },
|
||||||
|
randomToken: () => 'urgent-cookie',
|
||||||
|
now: () => clock,
|
||||||
|
runTurn: async () => {
|
||||||
|
hermesCalls += 1;
|
||||||
|
return gate.promise;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await service.unlock({ origin, accessCode });
|
||||||
|
|
||||||
|
// Saturate the single slot so any queued path would be rejected as overloaded.
|
||||||
|
const active = service.chat({ origin, cookieToken: 'urgent-cookie', payload: { message: 'normal question', ledger: [] } });
|
||||||
|
await new Promise(resolve => setImmediate(resolve));
|
||||||
|
assert.equal(hermesCalls, 1);
|
||||||
|
|
||||||
|
const urgent = await service.chat({ origin, cookieToken: 'urgent-cookie', payload: { message: 'I have rectal bleeding', ledger: [] } });
|
||||||
|
|
||||||
|
assert.equal(hermesCalls, 1, 'Hermes was called exactly once — only for the non-urgent request');
|
||||||
|
assert.equal(urgent.safetyOverride, true);
|
||||||
|
assert.match(urgent.reply, /medical help/i);
|
||||||
|
gate.resolve({ reply: 'late', sessionId: 'urgent_session_01' });
|
||||||
|
assert.match((await active).reply, /late/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vision analysis is bounded by the shared queue and overload returns the manual-fallback copy', async () => {
|
||||||
|
let clock = 0;
|
||||||
|
let providerCalls = 0;
|
||||||
|
const gates = [];
|
||||||
|
const queue = createInferenceQueue({ concurrency: 1, maxQueueDepth: 1, requestTimeoutMs: 60_000, now: () => clock });
|
||||||
|
|
||||||
|
const visionFetch = async (url, options) => {
|
||||||
|
providerCalls += 1;
|
||||||
|
const gate = deferred();
|
||||||
|
gates.push(gate);
|
||||||
|
options.signal?.addEventListener('abort', () => gate.reject(new Error('aborted')));
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
gate.promise.then(() => resolve({ ok: true, json: async () => ({ choices: [{ message: { content: JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.8, imageQuality: 'good', observations: 'ok' }) } }] }) }), reject);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const payload = { imageDataUrl: 'data:image/jpeg;base64,YQ==', consent: true };
|
||||||
|
const config = { baseUrl: 'http://127.0.0.1:8645/v1', apiKey: 'test-key-not-real', model: 'm' };
|
||||||
|
|
||||||
|
const first = queue.run(signal => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal }));
|
||||||
|
while (providerCalls < 1) await Promise.resolve();
|
||||||
|
const second = queue.run(signal => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal }));
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
await assert.rejects(() => queue.run((signal) => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal })), error => {
|
||||||
|
assert.ok(error instanceof OverloadError);
|
||||||
|
assert.match(error.message, /continue manually/i);
|
||||||
|
assert.doesNotMatch(error.message, /api|key|token|127\.0\.0\.1|fetch/i);
|
||||||
|
return true;
|
||||||
|
}, 'third request beyond depth is overloaded with sanitized copy');
|
||||||
|
|
||||||
|
assert.equal(providerCalls, 1, 'overloaded request never reached the provider');
|
||||||
|
gates[0].resolve('go');
|
||||||
|
const result = await Promise.race([first, new Promise(resolve => setTimeout(() => resolve('pending'), 100))]);
|
||||||
|
assert.notEqual(result, 'pending');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('vision requests carry cancellation so disconnected clients release provider work', async () => {
|
||||||
|
let capturedSignal;
|
||||||
|
const controller = new AbortController();
|
||||||
|
const gate = deferred();
|
||||||
|
const fetchImpl = async (url, options) => {
|
||||||
|
capturedSignal = options.signal;
|
||||||
|
options.signal?.addEventListener('abort', () => gate.reject(new Error('aborted')));
|
||||||
|
return gate.promise;
|
||||||
|
};
|
||||||
|
const payload = { imageDataUrl: 'data:image/jpeg;base64,YQ==', consent: true };
|
||||||
|
const config = { baseUrl: 'http://127.0.0.1:8645/v1', apiKey: 'test-key-not-real', model: 'm' };
|
||||||
|
|
||||||
|
const attempt = analyzePhoto({ payload, fetchImpl, config, signal: controller.signal });
|
||||||
|
while (!capturedSignal) await Promise.resolve();
|
||||||
|
|
||||||
|
controller.abort(new Error('client disconnected'));
|
||||||
|
await assert.rejects(() => attempt, /continue manually|unavailable/i);
|
||||||
|
assert.equal(capturedSignal.aborted, true, 'the provider fetch observed the cancellation');
|
||||||
|
});
|
||||||
93
tests/server-shutdown.acceptance.mjs
Normal file
93
tests/server-shutdown.acceptance.mjs
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { chmod, mkdtemp, rm, readFile } from 'node:fs/promises';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { constants as fsConstants } from 'node:fs';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const root = fileURLToPath(new URL('..', import.meta.url));
|
||||||
|
const fixture = fileURLToPath(new URL('./fixtures/slow-hermes.mjs', import.meta.url));
|
||||||
|
|
||||||
|
test('server shutdown cancels in-flight turns and leaves no orphan Hermes subprocesses', async () => {
|
||||||
|
const workdir = await mkdtemp(join(tmpdir(), 'timmy-shutdown-'));
|
||||||
|
await chmod(fixture, fsConstants.S_IRWXU);
|
||||||
|
const child = spawn(process.execPath, ['server.mjs'], {
|
||||||
|
cwd: root,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
HOST: '127.0.0.1',
|
||||||
|
PORT: '4188',
|
||||||
|
TIMMY_AGENT_ENABLED: 'true',
|
||||||
|
TIMMY_AGENT_ACCESS_TOKEN: 'shutdown-accept-code-2026',
|
||||||
|
TIMMY_PUBLIC_ORIGIN: 'http://127.0.0.1:4188',
|
||||||
|
TIMMY_AGENT_WORKDIR: workdir,
|
||||||
|
TIMMY_HERMES_COMMAND: fixture,
|
||||||
|
},
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
});
|
||||||
|
let reportedOrigin = '';
|
||||||
|
child.stdout.on('data', chunk => {
|
||||||
|
const match = String(chunk).match(/listening on http:\/\/([^:]+):(\d+)/);
|
||||||
|
if (match && !reportedOrigin) reportedOrigin = `http://${match[1]}:${match[2]}`;
|
||||||
|
});
|
||||||
|
const deadline = Date.now() + 10_000;
|
||||||
|
while (!reportedOrigin && Date.now() < deadline && child.exitCode === null) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
assert.ok(reportedOrigin, 'server reported its port');
|
||||||
|
|
||||||
|
const unlockResponse = await fetch(`${reportedOrigin}/api/agent/unlock`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json', origin: reportedOrigin, 'sec-fetch-site': 'same-origin' },
|
||||||
|
body: JSON.stringify({ accessCode: 'shutdown-accept-code-2026' }),
|
||||||
|
});
|
||||||
|
assert.equal(unlockResponse.status, 200);
|
||||||
|
const browserCookie = unlockResponse.headers.get('set-cookie').split(';', 1)[0];
|
||||||
|
|
||||||
|
// Start a turn wedged on the fixture's HOLD gate.
|
||||||
|
fetch(`${reportedOrigin}/api/agent/chat`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json', origin: reportedOrigin, 'sec-fetch-site': 'same-origin', cookie: browserCookie },
|
||||||
|
body: JSON.stringify({ message: 'HOLD shutdown probe', ledger: [] }),
|
||||||
|
}).catch(() => {});
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 600));
|
||||||
|
|
||||||
|
const pidLog = join(workdir, 'fixture-pids.log');
|
||||||
|
// Wait until the fixture for this run's HOLD turn is actually alive.
|
||||||
|
let lastPid = null;
|
||||||
|
const spawnDeadline = Date.now() + 8_000;
|
||||||
|
while (Date.now() < spawnDeadline) {
|
||||||
|
const recorded = ((await readFile(pidLog, 'utf8').catch(() => '')) || '')
|
||||||
|
.split('\n').map(Number).filter(Boolean);
|
||||||
|
const candidate = recorded.at(-1);
|
||||||
|
if (candidate) {
|
||||||
|
let alive = true;
|
||||||
|
try { process.kill(candidate, 0); } catch { alive = false; }
|
||||||
|
if (alive) {
|
||||||
|
lastPid = candidate;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 100));
|
||||||
|
}
|
||||||
|
assert.ok(lastPid, 'fixture subprocess was spawned for the in-flight turn');
|
||||||
|
let aliveBefore = true;
|
||||||
|
try { process.kill(lastPid, 0); } catch { aliveBefore = false; }
|
||||||
|
assert.ok(aliveBefore, `fixture ${lastPid} is running before shutdown`);
|
||||||
|
|
||||||
|
// Shut the server down while the turn is still running.
|
||||||
|
child.kill('SIGTERM');
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => reject(new Error('server did not exit after SIGTERM')), 5_000);
|
||||||
|
child.on('exit', code => { clearTimeout(timeout); resolve(code); });
|
||||||
|
});
|
||||||
|
|
||||||
|
// The in-flight fixture must be gone — no orphan subprocess may survive.
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 500));
|
||||||
|
let aliveAfter = true;
|
||||||
|
try { process.kill(lastPid, 0); } catch { aliveAfter = false; }
|
||||||
|
assert.equal(aliveAfter, false, `fixture process ${lastPid} survived server shutdown`);
|
||||||
|
await rm(workdir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}, { timeout: 30_000 });
|
||||||
Loading…
Reference in New Issue
Block a user