diff --git a/package.json b/package.json index 31839e0..8e4fba5 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,14 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-queue.test.js tests/agent-turn-timeout.test.js tests/inference-queue.test.js tests/inference-queue-cancellation.test.js tests/inference-zero-call.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:sleek": "node tests/sleek-chat.acceptance.mjs", + "test:queue-acceptance": "node tests/inference-queue.acceptance.mjs", + "test:shutdown-acceptance": "node tests/server-shutdown.acceptance.mjs", "test:staging-smoke": "node tests/staging.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/inference-queue.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/inference-queue.acceptance.mjs && node --check tests/server-shutdown.acceptance.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs" }, diff --git a/server.mjs b/server.mjs index 82f2ba1..34d569b 100644 --- a/server.mjs +++ b/server.mjs @@ -39,6 +39,34 @@ function rejectCrossSite(req){const site=String(req.headers['sec-fetch-site']||' function agentCookie(token){const secure=agentConfig.publicOrigin.startsWith('https://')?'; Secure':'';return `timmy_agent=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=${appRoot}; Max-Age=86400${secure}`} function sendAgentError(res,error){const status=error instanceof AgentGatewayError?error.status:503;const message=error instanceof AgentGatewayError?error.message:'Hermes is temporarily unavailable. Your local journal still works.';return sendJson(res,status,{error:message})} +// Graceful shutdown: SIGTERM must cancel in-flight Hermes turns (killing their +// child processes) instead of abandoning them as orphan subprocesses. +const inFlightChats = new Set(); +let shuttingDown = false; +function trackChat(res, controller) { + if (shuttingDown) controller.abort(new Error('Timmy is shutting down. Your journal still works — try again shortly.')); + const entry = { res, controller }; + inFlightChats.add(entry); + // A vanished client must release its queue slot and stop any in-flight + // Hermes turn instead of consuming inference capacity silently. + res.on('close', () => { + inFlightChats.delete(entry); + if (!res.writableEnded) { + try { controller.abort(new Error('The browser closed the request. Your journal still works.')); } catch { /* already aborted */ } + } + }); + return entry; +} +for (const signalName of ['SIGTERM', 'SIGINT']) { + process.on(signalName, () => { + shuttingDown = true; + for (const { controller } of inFlightChats) { + try { controller.abort(new Error('Server is shutting down.')); } catch { /* already aborted */ } + } + process.exit(0); + }); +} + http.createServer(async(req,res)=>{ try{ const url=new URL(req.url,'http://localhost'); @@ -62,7 +90,7 @@ http.createServer(async(req,res)=>{ try{rejectCrossSite(req);const payload=await readJson(req,4096);const result=await agentService.unlock({origin:requestOrigin(req),accessCode:String(payload?.accessCode||'')});res.setHeader('set-cookie',agentCookie(result.cookieToken));return sendJson(res,200,result.public)}catch(error){return sendAgentError(res,error)} } if(appPath==='/api/agent/chat'&&req.method==='POST'){ - try{rejectCrossSite(req);const payload=await readJson(req,128*1024);return sendJson(res,200,await agentService.chat({origin:requestOrigin(req),cookieToken:cookie(req,'timmy_agent'),payload}))}catch(error){return sendAgentError(res,error)} + try{rejectCrossSite(req);const payload=await readJson(req,128*1024);const chatAbort=new AbortController();trackChat(res,chatAbort);return sendJson(res,200,await agentService.chat({origin:requestOrigin(req),cookieToken:cookie(req,'timmy_agent'),payload,signal:chatAbort.signal}))}catch(error){return sendAgentError(res,error)} } if(appPath.startsWith('/api/'))return sendJson(res,404,{error:'Not found'}); if(req.method!=='GET'&&req.method!=='HEAD'){res.writeHead(405,{'allow':'GET, HEAD'});return res.end()} diff --git a/src/vision-service.js b/src/vision-service.js index 6edb789..d812ac3 100644 --- a/src/vision-service.js +++ b/src/vision-service.js @@ -7,18 +7,22 @@ function providerEndpoint(baseUrl) { return `${url.toString().replace(/\/$/, '')}/chat/completions`; } -export async function analyzePhoto({ payload, fetchImpl = fetch, config }) { +export async function analyzePhoto({ payload, fetchImpl = fetch, config, signal }) { const photo = validatePhotoPayload(payload); if (!config?.model) throw new Error('AI analysis is not configured.'); const endpoint = providerEndpoint(config.baseUrl); + // The provider call honours both its own deadline and a per-request + // cancellation so abandoned clients stop consuming inference capacity. + const timeoutSignal = AbortSignal.timeout(config.requestTimeoutMs || 60_000); + const combinedSignal = signal ? AbortSignal.any([timeoutSignal, signal]) : timeoutSignal; const response = await fetchImpl(endpoint, { method: 'POST', headers: { 'content-type': 'application/json', - authorization: `Bearer ${config.apiKey || 'local-proxy'}`, + authorization: 'Bearer ' + String(config.apiKey || 'local-proxy'), }, body: JSON.stringify(buildVisionRequest({ imageDataUrl: photo.imageDataUrl, model: config.model })), - signal: AbortSignal.timeout(config.requestTimeoutMs || 60_000), + signal: combinedSignal, }).catch(() => { throw new Error('AI analysis is temporarily unavailable. Continue manually.'); }); if (!response.ok) throw new Error('AI analysis is temporarily unavailable. Continue manually.'); let data; diff --git a/tests/fixtures/slow-hermes.mjs b/tests/fixtures/slow-hermes.mjs new file mode 100755 index 0000000..3580309 --- /dev/null +++ b/tests/fixtures/slow-hermes.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node +// Slow Hermes fixture for bounded-queue acceptance tests. Normal prompts +// answer immediately; prompts containing HOLD block until a release file +// appears inside the agent workdir (the value after `--in`), so tests can +// saturate the queue deterministically without touching shared /tmp state. +import { existsSync } from 'node:fs'; +import { appendFileSync } from 'node:fs'; + +const args = process.argv.slice(2); +const promptIndex = args.indexOf('-q'); +const prompt = promptIndex >= 0 ? args[promptIndex + 1] : ''; +const workdirIndex = args.indexOf('--in'); +const workdir = workdirIndex >= 0 ? args[workdirIndex + 1] : '/tmp'; +appendFileSync(`${workdir}/fixture-pids.log`, `${process.pid}\n`); +process.stdout.write('session_id: slow_fixture_2026\n'); +if (/HOLD/.test(prompt)) { + const releaseFile = `${workdir}/hermes-release`; + while (!existsSync(releaseFile)) await new Promise(resolve => setTimeout(resolve, 20)); +} +process.stdout.write('Hermes fixture answered the bounded request.\n'); diff --git a/tests/inference-queue.acceptance.mjs b/tests/inference-queue.acceptance.mjs new file mode 100644 index 0000000..f86bf67 --- /dev/null +++ b/tests/inference-queue.acceptance.mjs @@ -0,0 +1,136 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { chmod, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { constants as fsConstants } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const fixture = fileURLToPath(new URL('./fixtures/slow-hermes.mjs', import.meta.url)); +const origin = 'http://127.0.0.1:4187'; +const accessCode = 'queue-acceptance-code-2026'; + +async function waitReady(child) { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (child.exitCode !== null) throw new Error(`server exited ${child.exitCode}`); + try { const response = await fetch(`${origin}/api/healthz`); if (response.ok) return; } catch {} + await new Promise(resolve => setTimeout(resolve, 50)); + } + throw new Error('server did not become ready'); +} + +function post(path, body, { cookie = '', signal } = {}) { + return fetch(`${origin}${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + origin, + 'sec-fetch-site': 'same-origin', + ...(cookie ? { cookie } : {}), + }, + body: JSON.stringify(body), + signal, + }).then( + async response => ({ status: response.status, body: await response.json().catch(() => ({})) }), + error => ({ failed: true, reason: String(error?.name || error) }), + ); +} + +test('bounded queue end-to-end: disconnect cleanup, overload fallback, urgent bypass, recovery', async t => { + const workdir = await mkdtemp(join(tmpdir(), 'timmy-queue-accept-')); + const releaseFile = join(workdir, 'hermes-release'); + const pidLog = join(workdir, 'fixture-pids.log'); + await chmod(fixture, fsConstants.S_IRWXU); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env: { + ...process.env, + PORT: '4187', + TIMMY_AGENT_ENABLED: 'true', + TIMMY_AGENT_ACCESS_TOKEN: accessCode, + TIMMY_PUBLIC_ORIGIN: origin, + TIMMY_AGENT_WORKDIR: workdir, + TIMMY_HERMES_COMMAND: fixture, + TIMMY_AGENT_MAX_CONCURRENT_TURNS: '1', + TIMMY_AGENT_MAX_QUEUE_DEPTH: '2', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let serverLogs = ''; + child.stderr.on('data', chunk => { serverLogs += String(chunk); }); + t.after(async () => { + child.kill('SIGTERM'); + await rm(workdir, { recursive: true, force: true }).catch(() => {}); + await rm(releaseFile, { force: true }).catch(() => {}); + }); + await waitReady(child); + + // Authenticate once and keep the cookie for the whole scenario. + const rawResponse = await fetch(`${origin}/api/agent/unlock`, { + method: 'POST', + headers: { 'content-type': 'application/json', origin, 'sec-fetch-site': 'same-origin' }, + body: JSON.stringify({ accessCode }), + }); + assert.equal(rawResponse.status, 200); + const browserCookie = rawResponse.headers.get('set-cookie').split(';', 1)[0]; + + // Fill the slot: this HOLD runs until the fixture release file appears. + const heldOne = post('/api/agent/chat', { message: 'HOLD one', ledger: [] }, { cookie: browserCookie }); + await new Promise(resolve => setTimeout(resolve, 400)); + + // 1. Disconnect cleanup: this request waits in the queue. When its client + // vanishes, Timmy must drop it and free the queued place. + const disconnectController = new AbortController(); + const doomed = post('/api/agent/chat', { message: 'HOLD doomed', ledger: [] }, { cookie: browserCookie, signal: disconnectController.signal }); + await new Promise(resolve => setTimeout(resolve, 400)); + disconnectController.abort(); + const doomedOutcome = await doomed; + assert.equal(doomedOutcome.failed, true, 'abandoned client saw its request cancelled'); + + // If the freed place was reclaimed, both follow-up HOLDs are admitted + // (slot + queue of two minus one freed place). A sanitized 503 here would + // mean the cancelled client did not release capacity. + const heldTwo = post('/api/agent/chat', { message: 'HOLD two', ledger: [] }, { cookie: browserCookie }); + const heldThree = post('/api/agent/chat', { message: 'HOLD three', ledger: [] }, { cookie: browserCookie }); + await new Promise(resolve => setTimeout(resolve, 500)); + + // 2. Overload: everything is saturated now, so a further request must fail + // fast with the stable manual-fallback copy and zero provider contact. + const overloadResult = await post('/api/agent/chat', { message: 'overload probe', ledger: [] }, { cookie: browserCookie }); + assert.equal(overloadResult.status, 503); + assert.match(overloadResult.body.error, /busy|try again|manually/i); + assert.doesNotMatch(overloadResult.body.error, /spawn|hermes|session|token|workdir|fixture/i); + + // 3. Urgent bypass while saturated: deterministic safety answer, zero calls. + const urgentResult = await post('/api/agent/chat', { message: 'I have rectal bleeding', ledger: [] }, { cookie: browserCookie }); + assert.equal(urgentResult.status, 200); + assert.equal(urgentResult.body.safetyOverride, true); + assert.match(urgentResult.body.reply, /medical help/i); + + // 4. Recovery: releasing the fixture lets every surviving request complete. + await writeFile(releaseFile, 'go'); + for (const [label, promise] of [['one', heldOne], ['two', heldTwo], ['three', heldThree]]) { + const settled = await Promise.race([ + promise, + new Promise(resolve => setTimeout(() => resolve({ status: 'TEST-TIMEOUT' }), 12_000)), + ]); + assert.equal(settled.status, 200, `${label} completed after recovery`); + assert.match(settled.body.reply, /bounded request/); + assert.doesNotMatch(JSON.stringify(settled.body), /slow_fixture_2026|session_id/); + } + + // 5. No orphan subprocesses: every fixture PID recorded at spawn is gone. + await new Promise(resolve => setTimeout(resolve, 400)); + const recordedPids = ((await readFile(pidLog, 'utf8').catch(() => '')) || '') + .split('\n').map(Number).filter(Boolean); + assert.equal(recordedPids.length, 3, `exactly three turns ran (saw ${recordedPids.length})`); + for (const pid of [...new Set(recordedPids)]) { + let alive = true; + try { process.kill(pid, 0); } catch { alive = false; } + assert.equal(alive, false, `fixture process ${pid} outlived its turn`); + } + assert.doesNotMatch(serverLogs, /access-code|secret|token/i); +}, { timeout: 40_000 }); diff --git a/tests/inference-zero-call.test.js b/tests/inference-zero-call.test.js new file mode 100644 index 0000000..1218a8f --- /dev/null +++ b/tests/inference-zero-call.test.js @@ -0,0 +1,108 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { createHermesAgentService, resolveHermesAgentConfig } from '../src/hermes-agent-service.js'; +import { analyzePhoto } from '../src/vision-service.js'; +import { OverloadError, createInferenceQueue } from '../src/inference-queue.js'; + +const origin = 'http://127.0.0.1:4173'; +const accessCode = 'test-agent-access-code-2026'; +const configured = () => resolveHermesAgentConfig({ + TIMMY_AGENT_ENABLED: 'true', + TIMMY_AGENT_ACCESS_TOKEN: accessCode, + TIMMY_PUBLIC_ORIGIN: origin, + TIMMY_AGENT_WORKDIR: '/tmp/timmy-agent-workspace', +}); + +function deferred() { + let resolve; + let reject; + const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); + return { promise, resolve, reject }; +} + +test('urgent messages intercept before Hermes even when the queue is saturated', async () => { + let clock = 0; + let hermesCalls = 0; + const gate = deferred(); + const service = createHermesAgentService({ + config: { ...configured(), maxQueueDepth: 0 }, + randomToken: () => 'urgent-cookie', + now: () => clock, + runTurn: async () => { + hermesCalls += 1; + return gate.promise; + }, + }); + await service.unlock({ origin, accessCode }); + + // Saturate the single slot so any queued path would be rejected as overloaded. + const active = service.chat({ origin, cookieToken: 'urgent-cookie', payload: { message: 'normal question', ledger: [] } }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(hermesCalls, 1); + + const urgent = await service.chat({ origin, cookieToken: 'urgent-cookie', payload: { message: 'I have rectal bleeding', ledger: [] } }); + + assert.equal(hermesCalls, 1, 'Hermes was called exactly once — only for the non-urgent request'); + assert.equal(urgent.safetyOverride, true); + assert.match(urgent.reply, /medical help/i); + gate.resolve({ reply: 'late', sessionId: 'urgent_session_01' }); + assert.match((await active).reply, /late/); +}); + +test('vision analysis is bounded by the shared queue and overload returns the manual-fallback copy', async () => { + let clock = 0; + let providerCalls = 0; + const gates = []; + const queue = createInferenceQueue({ concurrency: 1, maxQueueDepth: 1, requestTimeoutMs: 60_000, now: () => clock }); + + const visionFetch = async (url, options) => { + providerCalls += 1; + const gate = deferred(); + gates.push(gate); + options.signal?.addEventListener('abort', () => gate.reject(new Error('aborted'))); + return new Promise((resolve, reject) => { + gate.promise.then(() => resolve({ ok: true, json: async () => ({ choices: [{ message: { content: JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.8, imageQuality: 'good', observations: 'ok' }) } }] }) }), reject); + }); + }; + + const payload = { imageDataUrl: 'data:image/jpeg;base64,YQ==', consent: true }; + const config = { baseUrl: 'http://127.0.0.1:8645/v1', apiKey: 'test-key-not-real', model: 'm' }; + + const first = queue.run(signal => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal })); + while (providerCalls < 1) await Promise.resolve(); + const second = queue.run(signal => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal })); + await Promise.resolve(); + + await assert.rejects(() => queue.run((signal) => analyzePhoto({ payload, fetchImpl: visionFetch, config, signal })), error => { + assert.ok(error instanceof OverloadError); + assert.match(error.message, /continue manually/i); + assert.doesNotMatch(error.message, /api|key|token|127\.0\.0\.1|fetch/i); + return true; + }, 'third request beyond depth is overloaded with sanitized copy'); + + assert.equal(providerCalls, 1, 'overloaded request never reached the provider'); + gates[0].resolve('go'); + const result = await Promise.race([first, new Promise(resolve => setTimeout(() => resolve('pending'), 100))]); + assert.notEqual(result, 'pending'); +}); + +test('vision requests carry cancellation so disconnected clients release provider work', async () => { + let capturedSignal; + const controller = new AbortController(); + const gate = deferred(); + const fetchImpl = async (url, options) => { + capturedSignal = options.signal; + options.signal?.addEventListener('abort', () => gate.reject(new Error('aborted'))); + return gate.promise; + }; + const payload = { imageDataUrl: 'data:image/jpeg;base64,YQ==', consent: true }; + const config = { baseUrl: 'http://127.0.0.1:8645/v1', apiKey: 'test-key-not-real', model: 'm' }; + + const attempt = analyzePhoto({ payload, fetchImpl, config, signal: controller.signal }); + while (!capturedSignal) await Promise.resolve(); + + controller.abort(new Error('client disconnected')); + await assert.rejects(() => attempt, /continue manually|unavailable/i); + assert.equal(capturedSignal.aborted, true, 'the provider fetch observed the cancellation'); +}); diff --git a/tests/server-shutdown.acceptance.mjs b/tests/server-shutdown.acceptance.mjs new file mode 100644 index 0000000..0f61f79 --- /dev/null +++ b/tests/server-shutdown.acceptance.mjs @@ -0,0 +1,93 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { chmod, mkdtemp, rm, readFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { constants as fsConstants } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const fixture = fileURLToPath(new URL('./fixtures/slow-hermes.mjs', import.meta.url)); + +test('server shutdown cancels in-flight turns and leaves no orphan Hermes subprocesses', async () => { + const workdir = await mkdtemp(join(tmpdir(), 'timmy-shutdown-')); + await chmod(fixture, fsConstants.S_IRWXU); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env: { + ...process.env, + HOST: '127.0.0.1', + PORT: '4188', + TIMMY_AGENT_ENABLED: 'true', + TIMMY_AGENT_ACCESS_TOKEN: 'shutdown-accept-code-2026', + TIMMY_PUBLIC_ORIGIN: 'http://127.0.0.1:4188', + TIMMY_AGENT_WORKDIR: workdir, + TIMMY_HERMES_COMMAND: fixture, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let reportedOrigin = ''; + child.stdout.on('data', chunk => { + const match = String(chunk).match(/listening on http:\/\/([^:]+):(\d+)/); + if (match && !reportedOrigin) reportedOrigin = `http://${match[1]}:${match[2]}`; + }); + const deadline = Date.now() + 10_000; + while (!reportedOrigin && Date.now() < deadline && child.exitCode === null) { + await new Promise(resolve => setTimeout(resolve, 50)); + } + assert.ok(reportedOrigin, 'server reported its port'); + + const unlockResponse = await fetch(`${reportedOrigin}/api/agent/unlock`, { + method: 'POST', + headers: { 'content-type': 'application/json', origin: reportedOrigin, 'sec-fetch-site': 'same-origin' }, + body: JSON.stringify({ accessCode: 'shutdown-accept-code-2026' }), + }); + assert.equal(unlockResponse.status, 200); + const browserCookie = unlockResponse.headers.get('set-cookie').split(';', 1)[0]; + + // Start a turn wedged on the fixture's HOLD gate. + fetch(`${reportedOrigin}/api/agent/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json', origin: reportedOrigin, 'sec-fetch-site': 'same-origin', cookie: browserCookie }, + body: JSON.stringify({ message: 'HOLD shutdown probe', ledger: [] }), + }).catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 600)); + + const pidLog = join(workdir, 'fixture-pids.log'); + // Wait until the fixture for this run's HOLD turn is actually alive. + let lastPid = null; + const spawnDeadline = Date.now() + 8_000; + while (Date.now() < spawnDeadline) { + const recorded = ((await readFile(pidLog, 'utf8').catch(() => '')) || '') + .split('\n').map(Number).filter(Boolean); + const candidate = recorded.at(-1); + if (candidate) { + let alive = true; + try { process.kill(candidate, 0); } catch { alive = false; } + if (alive) { + lastPid = candidate; + break; + } + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + assert.ok(lastPid, 'fixture subprocess was spawned for the in-flight turn'); + let aliveBefore = true; + try { process.kill(lastPid, 0); } catch { aliveBefore = false; } + assert.ok(aliveBefore, `fixture ${lastPid} is running before shutdown`); + + // Shut the server down while the turn is still running. + child.kill('SIGTERM'); + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('server did not exit after SIGTERM')), 5_000); + child.on('exit', code => { clearTimeout(timeout); resolve(code); }); + }); + + // The in-flight fixture must be gone — no orphan subprocess may survive. + await new Promise(resolve => setTimeout(resolve, 500)); + let aliveAfter = true; + try { process.kill(lastPid, 0); } catch { aliveAfter = false; } + assert.equal(aliveAfter, false, `fixture process ${lastPid} survived server shutdown`); + await rm(workdir, { recursive: true, force: true }).catch(() => {}); +}, { timeout: 30_000 });