feat: quality analysis — bug fixes, mobile tests, HITL checklist

Senior architect review findings + remediations:

BUG FIX — critical interface mismatch
- TimmyAirLLMAgent only exposed print_response(); dashboard route calls
  agent.run() → AttributeError when AirLLM backend is selected.
  Added run() → RunResult(content) as primary inference entry point;
  print_response() now delegates to run() so both call sites share
  one inference path.
- Added RunResult dataclass for Agno-compatible structured return.

BUG FIX — hardcoded model name in health status partial
- health_status.html rendered literal "llama3.2" regardless of
  OLLAMA_MODEL env var. Route now passes settings.ollama_model to
  the template context; partial renders {{ model }} instead.

FEATURE — /mobile-test HITL checklist page
- 22 human-executable test scenarios across: Layout, Touch & Input,
  Chat behaviour, Health, Scroll, Notch/Home Bar, Live UI.
- Pass/Fail/Skip buttons with sessionStorage state persistence.
- Live progress bar + final score summary.
- TEST link added to Mission Control header for quick access on phone.

TEST — 32 new automated mobile quality tests (M1xx–M6xx)
- M1xx: viewport/meta tags (8 tests)
- M2xx: touch target sizing — 44 px min-height, manipulation (4 tests)
- M3xx: iOS zoom prevention, autocapitalize, enterkeyhint (5 tests)
- M4xx: HTMX robustness — hx-sync drop, disabled-elt, polling (5 tests)
- M5xx: safe-area insets, overscroll, dvh units (5 tests)
- M6xx: AirLLM interface contract — run(), RunResult, delegation (5 tests)

Total test count: 61 → 93 (all passing).

https://claude.ai/code/session_01RBuRCBXZNkAQQXXGiJNDmt
This commit is contained in:
Claude
2026-02-21 17:21:47 +00:00
parent 7499690e10
commit c8aa6a5fbb
9 changed files with 958 additions and 12 deletions

View File

@@ -9,6 +9,7 @@ from fastapi.templating import Jinja2Templates
from config import settings
from dashboard.routes.agents import router as agents_router
from dashboard.routes.health import router as health_router
from dashboard.routes.mobile_test import router as mobile_test_router
logging.basicConfig(
level=logging.INFO,
@@ -33,6 +34,7 @@ app.mount("/static", StaticFiles(directory=str(PROJECT_ROOT / "static")), name="
app.include_router(health_router)
app.include_router(agents_router)
app.include_router(mobile_test_router)
@app.get("/", response_class=HTMLResponse)

View File

@@ -38,5 +38,5 @@ async def health_status(request: Request):
return templates.TemplateResponse(
request,
"partials/health_status.html",
{"ollama": ollama_ok},
{"ollama": ollama_ok, "model": settings.ollama_model},
)

View File

@@ -0,0 +1,257 @@
"""Mobile HITL (Human-in-the-Loop) test checklist route.
GET /mobile-test — interactive checklist for a human tester on their phone.
Each scenario specifies what to do and what to observe. The tester marks
each one PASS / FAIL / SKIP. Results are stored in sessionStorage so they
survive page scrolling without hitting the server.
"""
from pathlib import Path
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
router = APIRouter(tags=["mobile-test"])
templates = Jinja2Templates(directory=str(Path(__file__).parent.parent / "templates"))
# ── Test scenarios ────────────────────────────────────────────────────────────
# Each dict: id, category, title, steps (list), expected
SCENARIOS = [
# Layout
{
"id": "L01",
"category": "Layout",
"title": "Sidebar renders as horizontal strip",
"steps": [
"Open the Mission Control page on your phone.",
"Look at the top section above the chat window.",
],
"expected": (
"AGENTS and SYSTEM HEALTH panels appear side-by-side in a "
"horizontally scrollable strip — not stacked vertically."
),
},
{
"id": "L02",
"category": "Layout",
"title": "Sidebar panels are horizontally scrollable",
"steps": [
"Swipe left/right on the AGENTS / SYSTEM HEALTH strip.",
],
"expected": "Both panels slide smoothly; no page scroll is triggered.",
},
{
"id": "L03",
"category": "Layout",
"title": "Chat panel fills ≥ 60 % of viewport height",
"steps": [
"Look at the TIMMY INTERFACE chat card below the strip.",
],
"expected": "The chat card occupies at least 60 % of the visible screen height.",
},
{
"id": "L04",
"category": "Layout",
"title": "Header stays fixed while chat scrolls",
"steps": [
"Send several messages until the chat overflows.",
"Scroll the chat log up and down.",
],
"expected": "The TIMMY TIME / MISSION CONTROL header remains pinned at the top.",
},
{
"id": "L05",
"category": "Layout",
"title": "No horizontal page overflow",
"steps": [
"Try swiping left or right anywhere on the page.",
],
"expected": "The page does not scroll horizontally; nothing is cut off.",
},
# Touch & Input
{
"id": "T01",
"category": "Touch & Input",
"title": "iOS does NOT zoom when tapping the input",
"steps": [
"Tap the message input field once.",
"Watch whether the browser zooms in.",
],
"expected": "The keyboard rises; the layout does NOT zoom in.",
},
{
"id": "T02",
"category": "Touch & Input",
"title": "Keyboard return key is labelled 'Send'",
"steps": [
"Tap the message input to open the iOS/Android keyboard.",
"Look at the return / action key in the bottom-right of the keyboard.",
],
"expected": "The key is labelled 'Send' (not 'Return' or 'Go').",
},
{
"id": "T03",
"category": "Touch & Input",
"title": "Send button is easy to tap (≥ 44 px tall)",
"steps": [
"Try tapping the SEND button with your thumb.",
],
"expected": "The button registers the tap reliably on the first attempt.",
},
{
"id": "T04",
"category": "Touch & Input",
"title": "SEND button disabled during in-flight request",
"steps": [
"Type a message and press SEND.",
"Immediately try to tap SEND again before a response arrives.",
],
"expected": "The button is visually disabled; no duplicate message is sent.",
},
{
"id": "T05",
"category": "Touch & Input",
"title": "Empty message cannot be submitted",
"steps": [
"Leave the input blank.",
"Tap SEND.",
],
"expected": "Nothing is submitted; the form shows a required-field indicator.",
},
{
"id": "T06",
"category": "Touch & Input",
"title": "CLEAR button shows confirmation dialog",
"steps": [
"Send at least one message.",
"Tap the CLEAR button in the top-right of the chat header.",
],
"expected": "A browser confirmation dialog appears before history is cleared.",
},
# Chat behaviour
{
"id": "C01",
"category": "Chat",
"title": "Chat auto-scrolls to the latest message",
"steps": [
"Scroll the chat log to the top.",
"Send a new message.",
],
"expected": "After the response arrives the chat automatically scrolls to the bottom.",
},
{
"id": "C02",
"category": "Chat",
"title": "Multi-turn conversation — Timmy remembers context",
"steps": [
"Send: 'My name is <your name>.'",
"Then send: 'What is my name?'",
],
"expected": "Timmy replies with your name, demonstrating conversation memory.",
},
{
"id": "C03",
"category": "Chat",
"title": "Loading indicator appears while waiting",
"steps": [
"Send a message and watch the SEND button.",
],
"expected": "A blinking cursor (▋) appears next to SEND while the response is loading.",
},
{
"id": "C04",
"category": "Chat",
"title": "Offline error is shown gracefully",
"steps": [
"Stop Ollama on your host machine (or disconnect from Wi-Fi temporarily).",
"Send a message from your phone.",
],
"expected": "A red 'Timmy is offline' error appears in the chat — no crash or spinner hang.",
},
# Health panel
{
"id": "H01",
"category": "Health",
"title": "Health panel shows Ollama UP when running",
"steps": [
"Ensure Ollama is running on your host.",
"Check the SYSTEM HEALTH panel.",
],
"expected": "OLLAMA badge shows green UP.",
},
{
"id": "H02",
"category": "Health",
"title": "Health panel auto-refreshes without reload",
"steps": [
"Start Ollama if it is not running.",
"Wait up to 35 seconds with the page open.",
],
"expected": "The OLLAMA badge flips from DOWN → UP automatically, without a page reload.",
},
# Scroll & overscroll
{
"id": "S01",
"category": "Scroll",
"title": "No rubber-band / bounce on the main page",
"steps": [
"Scroll to the very top of the page.",
"Continue pulling downward.",
],
"expected": "The page does not bounce or show a white gap — overscroll is suppressed.",
},
{
"id": "S02",
"category": "Scroll",
"title": "Chat log scrolls independently inside the card",
"steps": [
"Scroll inside the chat log area.",
],
"expected": "The chat log scrolls smoothly; the outer page does not move.",
},
# Safe area / notch
{
"id": "N01",
"category": "Notch / Home Bar",
"title": "Header clears the status bar / Dynamic Island",
"steps": [
"On a notched iPhone (Face ID), look at the top of the page.",
],
"expected": "The TIMMY TIME header text is not obscured by the notch or Dynamic Island.",
},
{
"id": "N02",
"category": "Notch / Home Bar",
"title": "Chat input not hidden behind home indicator",
"steps": [
"Tap the input field and look at the bottom of the screen.",
],
"expected": "The input row sits above the iPhone home indicator bar — nothing is cut off.",
},
# Clock
{
"id": "X01",
"category": "Live UI",
"title": "Clock updates every second",
"steps": [
"Look at the time display in the top-right of the header.",
"Watch for 3 seconds.",
],
"expected": "The time increments each second in HH:MM:SS format.",
},
]
@router.get("/mobile-test", response_class=HTMLResponse)
async def mobile_test(request: Request):
"""Interactive HITL mobile test checklist — open on your phone."""
categories: dict[str, list] = {}
for s in SCENARIOS:
categories.setdefault(s["category"], []).append(s)
return templates.TemplateResponse(
request,
"mobile_test.html",
{"scenarios": SCENARIOS, "categories": categories, "total": len(SCENARIOS)},
)

View File

@@ -21,6 +21,7 @@
<span class="mc-subtitle">MISSION CONTROL</span>
</div>
<div class="mc-header-right">
<a href="/mobile-test" class="mc-test-link">TEST</a>
<span class="mc-time" id="clock"></span>
</div>
</header>

View File

@@ -0,0 +1,375 @@
{% extends "base.html" %}
{% block title %}Mobile Test — Timmy Time{% endblock %}
{% block content %}
<div class="container-fluid mc-content" style="height:auto; overflow:visible;">
<!-- ── Page header ─────────────────────────────────────────────────── -->
<div class="mt-hitl-header">
<div>
<span class="mt-title">// MOBILE TEST SUITE</span>
<span class="mt-sub">HUMAN-IN-THE-LOOP</span>
</div>
<div class="mt-score-wrap">
<span class="mt-score" id="score-display">0 / {{ total }}</span>
<span class="mt-score-label">PASSED</span>
</div>
</div>
<!-- ── Progress bar ────────────────────────────────────────────────── -->
<div class="mt-progress-wrap">
<div class="progress" style="height:6px; background:var(--bg-card); border-radius:3px;">
<div class="progress-bar mt-progress-bar"
id="progress-bar"
role="progressbar"
style="width:0%; background:var(--green);"
aria-valuenow="0" aria-valuemin="0" aria-valuemax="{{ total }}"></div>
</div>
<div class="mt-progress-legend">
<span><span class="mt-dot green"></span>PASS</span>
<span><span class="mt-dot red"></span>FAIL</span>
<span><span class="mt-dot amber"></span>SKIP</span>
<span><span class="mt-dot" style="background:var(--text-dim);box-shadow:none;"></span>PENDING</span>
</div>
</div>
<!-- ── Reset / Back ────────────────────────────────────────────────── -->
<div class="mt-actions">
<a href="/" class="mc-btn-clear">← MISSION CONTROL</a>
<button class="mc-btn-clear" onclick="resetAll()" style="border-color:var(--red);color:var(--red);">RESET ALL</button>
</div>
<!-- ── Scenario cards ──────────────────────────────────────────────── -->
{% for category, items in categories.items() %}
<div class="mt-category-label">{{ category | upper }}</div>
{% for s in items %}
<div class="card mc-panel mt-card" id="card-{{ s.id }}" data-scenario="{{ s.id }}">
<div class="card-header mc-panel-header d-flex justify-content-between align-items-center">
<div>
<span class="mt-id-badge" id="badge-{{ s.id }}">{{ s.id }}</span>
<span class="mt-scenario-title">{{ s.title }}</span>
</div>
<span class="mt-state-chip" id="chip-{{ s.id }}">PENDING</span>
</div>
<div class="card-body p-3">
<div class="mt-steps-label">STEPS</div>
<ol class="mt-steps">
{% for step in s.steps %}
<li>{{ step }}</li>
{% endfor %}
</ol>
<div class="mt-expected-label">EXPECTED</div>
<div class="mt-expected">{{ s.expected }}</div>
<div class="mt-btn-row">
<button class="mt-btn mt-btn-pass" onclick="mark('{{ s.id }}', 'pass')">✓ PASS</button>
<button class="mt-btn mt-btn-fail" onclick="mark('{{ s.id }}', 'fail')">✗ FAIL</button>
<button class="mt-btn mt-btn-skip" onclick="mark('{{ s.id }}', 'skip')">— SKIP</button>
</div>
</div>
</div>
{% endfor %}
{% endfor %}
<!-- ── Summary footer ──────────────────────────────────────────────── -->
<div class="card mc-panel mt-summary" id="summary">
<div class="card-header mc-panel-header">// SUMMARY</div>
<div class="card-body p-3" id="summary-body">
<p class="mt-summary-hint">Mark all scenarios above to see your final score.</p>
</div>
</div>
</div><!-- /container -->
<!-- ── Styles (scoped to this page) ────────────────────────────────────── -->
<style>
.mt-hitl-header {
display: flex;
justify-content: space-between;
align-items: flex-end;
padding: 16px 0 12px;
border-bottom: 1px solid var(--border);
margin-bottom: 12px;
}
.mt-title {
font-size: 14px;
font-weight: 700;
color: var(--text-bright);
letter-spacing: 0.18em;
display: block;
}
.mt-sub {
font-size: 10px;
color: var(--text-dim);
letter-spacing: 0.2em;
display: block;
margin-top: 2px;
}
.mt-score-wrap { text-align: right; }
.mt-score {
font-size: 22px;
font-weight: 700;
color: var(--green);
letter-spacing: 0.06em;
display: block;
}
.mt-score-label { font-size: 9px; color: var(--text-dim); letter-spacing: 0.2em; }
.mt-progress-wrap { margin-bottom: 10px; }
.mt-progress-legend {
display: flex;
gap: 16px;
font-size: 9px;
color: var(--text-dim);
letter-spacing: 0.12em;
margin-top: 6px;
}
.mt-dot {
display: inline-block;
width: 7px;
height: 7px;
border-radius: 50%;
margin-right: 4px;
vertical-align: middle;
}
.mt-dot.green { background: var(--green); box-shadow: 0 0 5px var(--green); }
.mt-dot.red { background: var(--red); box-shadow: 0 0 5px var(--red); }
.mt-dot.amber { background: var(--amber); box-shadow: 0 0 5px var(--amber); }
.mt-actions {
display: flex;
gap: 10px;
margin-bottom: 16px;
}
.mt-category-label {
font-size: 9px;
font-weight: 700;
color: var(--text-dim);
letter-spacing: 0.25em;
margin: 20px 0 8px;
padding-left: 2px;
}
.mt-card {
margin-bottom: 10px;
transition: border-color 0.2s;
}
.mt-card.state-pass { border-color: var(--green) !important; }
.mt-card.state-fail { border-color: var(--red) !important; }
.mt-card.state-skip { border-color: var(--amber) !important; opacity: 0.7; }
.mt-id-badge {
font-size: 9px;
font-weight: 700;
background: var(--border);
color: var(--text-dim);
border-radius: 2px;
padding: 2px 6px;
letter-spacing: 0.12em;
margin-right: 8px;
}
.mt-card.state-pass .mt-id-badge { background: var(--green-dim); color: var(--green); }
.mt-card.state-fail .mt-id-badge { background: var(--red-dim); color: var(--red); }
.mt-card.state-skip .mt-id-badge { background: var(--amber-dim); color: var(--amber); }
.mt-scenario-title {
font-size: 12px;
font-weight: 700;
color: var(--text-bright);
letter-spacing: 0.05em;
}
.mt-state-chip {
font-size: 9px;
font-weight: 700;
letter-spacing: 0.15em;
color: var(--text-dim);
padding: 2px 8px;
border: 1px solid var(--border);
border-radius: 2px;
white-space: nowrap;
}
.mt-card.state-pass .mt-state-chip { color: var(--green); border-color: var(--green); }
.mt-card.state-fail .mt-state-chip { color: var(--red); border-color: var(--red); }
.mt-card.state-skip .mt-state-chip { color: var(--amber); border-color: var(--amber); }
.mt-steps-label, .mt-expected-label {
font-size: 9px;
font-weight: 700;
color: var(--text-dim);
letter-spacing: 0.2em;
margin-bottom: 6px;
}
.mt-expected-label { margin-top: 12px; }
.mt-steps {
padding-left: 18px;
margin: 0;
font-size: 12px;
line-height: 1.8;
color: var(--text);
}
.mt-expected {
font-size: 12px;
line-height: 1.65;
color: var(--text-bright);
background: var(--bg-card);
border-left: 3px solid var(--border-glow);
padding: 8px 12px;
border-radius: 0 3px 3px 0;
}
.mt-btn-row {
display: flex;
gap: 8px;
margin-top: 14px;
}
.mt-btn {
flex: 1;
min-height: 44px;
border: 1px solid var(--border);
border-radius: 3px;
background: var(--bg-deep);
color: var(--text-dim);
font-family: var(--font);
font-size: 11px;
font-weight: 700;
letter-spacing: 0.12em;
cursor: pointer;
touch-action: manipulation;
transition: background 0.15s, color 0.15s, border-color 0.15s;
}
.mt-btn-pass:hover, .mt-btn-pass.active { background: var(--green-dim); color: var(--green); border-color: var(--green); }
.mt-btn-fail:hover, .mt-btn-fail.active { background: var(--red-dim); color: var(--red); border-color: var(--red); }
.mt-btn-skip:hover, .mt-btn-skip.active { background: var(--amber-dim); color: var(--amber); border-color: var(--amber); }
.mt-summary { margin-top: 24px; margin-bottom: 32px; }
.mt-summary-hint { color: var(--text-dim); font-size: 12px; margin: 0; }
.mt-summary-row {
display: flex;
justify-content: space-between;
align-items: center;
padding: 8px 0;
border-bottom: 1px solid var(--border);
font-size: 12px;
}
.mt-summary-row:last-child { border-bottom: none; }
.mt-summary-score { font-size: 28px; font-weight: 700; color: var(--green); margin: 12px 0 4px; }
.mt-summary-pct { font-size: 13px; color: var(--text-dim); }
@media (max-width: 768px) {
.mt-btn-row { gap: 6px; }
.mt-btn { font-size: 10px; padding: 0 4px; }
}
</style>
<!-- ── HITL State Machine (sessionStorage) ─────────────────────────────── -->
<script>
const TOTAL = {{ total }};
const KEY = "timmy-mobile-test-results";
function loadResults() {
try { return JSON.parse(sessionStorage.getItem(KEY) || "{}"); }
catch { return {}; }
}
function saveResults(r) {
sessionStorage.setItem(KEY, JSON.stringify(r));
}
function mark(id, state) {
const results = loadResults();
results[id] = state;
saveResults(results);
applyState(id, state);
updateScore(results);
updateSummary(results);
}
function applyState(id, state) {
const card = document.getElementById("card-" + id);
const chip = document.getElementById("chip-" + id);
const labels = { pass: "PASS", fail: "FAIL", skip: "SKIP" };
card.classList.remove("state-pass", "state-fail", "state-skip");
if (state) card.classList.add("state-" + state);
chip.textContent = state ? labels[state] : "PENDING";
// highlight active button
card.querySelectorAll(".mt-btn").forEach(btn => btn.classList.remove("active"));
const activeBtn = card.querySelector(".mt-btn-" + state);
if (activeBtn) activeBtn.classList.add("active");
}
function updateScore(results) {
const passed = Object.values(results).filter(v => v === "pass").length;
const decided = Object.values(results).filter(v => v !== undefined).length;
document.getElementById("score-display").textContent = passed + " / " + TOTAL;
const pct = TOTAL ? (decided / TOTAL) * 100 : 0;
const bar = document.getElementById("progress-bar");
bar.style.width = pct + "%";
// colour the bar by overall health
const failCount = Object.values(results).filter(v => v === "fail").length;
bar.style.background = failCount > 0
? "var(--red)"
: passed === TOTAL ? "var(--green)" : "var(--amber)";
}
function updateSummary(results) {
const passed = Object.values(results).filter(v => v === "pass").length;
const failed = Object.values(results).filter(v => v === "fail").length;
const skipped = Object.values(results).filter(v => v === "skip").length;
const decided = passed + failed + skipped;
if (decided < TOTAL) {
document.getElementById("summary-body").innerHTML =
'<p class="mt-summary-hint">' + (TOTAL - decided) + ' scenario(s) still pending.</p>';
return;
}
const pct = TOTAL ? Math.round((passed / TOTAL) * 100) : 0;
const color = failed > 0 ? "var(--red)" : "var(--green)";
document.getElementById("summary-body").innerHTML = `
<div class="mt-summary-score" style="color:${color}">${passed} / ${TOTAL}</div>
<div class="mt-summary-pct">${pct}% pass rate</div>
<div style="margin-top:16px;">
<div class="mt-summary-row"><span>PASSED</span><span style="color:var(--green);font-weight:700;">${passed}</span></div>
<div class="mt-summary-row"><span>FAILED</span><span style="color:var(--red);font-weight:700;">${failed}</span></div>
<div class="mt-summary-row"><span>SKIPPED</span><span style="color:var(--amber);font-weight:700;">${skipped}</span></div>
</div>
${failed > 0 ? '<p style="color:var(--red);margin-top:12px;font-size:11px;">⚠ ' + failed + ' failure(s) need attention before release.</p>' : '<p style="color:var(--green);margin-top:12px;font-size:11px;">All tested scenarios passed — ship it.</p>'}
`;
}
function resetAll() {
if (!confirm("Reset all test results?")) return;
sessionStorage.removeItem(KEY);
const results = {};
document.querySelectorAll("[data-scenario]").forEach(card => {
const id = card.dataset.scenario;
applyState(id, null);
});
updateScore(results);
document.getElementById("summary-body").innerHTML =
'<p class="mt-summary-hint">Mark all scenarios above to see your final score.</p>';
}
// Restore saved state on load
(function init() {
const results = loadResults();
Object.entries(results).forEach(([id, state]) => applyState(id, state));
updateScore(results);
updateSummary(results);
})();
</script>
{% endblock %}

View File

@@ -14,6 +14,6 @@
</div>
<div class="health-row">
<span class="health-label">MODEL</span>
<span class="badge mc-badge-ready">llama3.2</span>
<span class="badge mc-badge-ready">{{ model }}</span>
</div>
</div>