diff --git a/frontend/dashboard.css b/frontend/dashboard.css index 99158d8..66cc8bf 100644 --- a/frontend/dashboard.css +++ b/frontend/dashboard.css @@ -673,6 +673,12 @@ textarea { resize: vertical; min-height: 120px; } .create-issue-header { display:flex; align-items:center; justify-content:space-between; gap:10px; } .create-issue-header button, .create-issue-actions button, .create-issue-capture-actions button { min-height:44px; } .create-issue-form { display:grid; gap:12px; } +.voice-issue-capture { display:grid; gap:8px; min-width:0; padding:12px; border:1px solid #31577f; border-radius:12px; background:#101d31; } +.voice-issue-capture[hidden], .voice-issue-review[hidden], .voice-issue-controls button[hidden] { display:none; } +.voice-issue-heading, .voice-issue-review { display:grid; gap:6px; } +.voice-issue-controls, .voice-issue-review-actions { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:8px; } +.voice-issue-controls button, .voice-issue-review-actions button { min-width:0; min-height:44px; } +#voice-issue-transcript { box-sizing:border-box; width:100%; min-height:96px; resize:vertical; } .create-issue-capture-actions { position:sticky; bottom:0; display:grid; grid-template-columns:1fr 1fr; gap:8px; padding:10px 0; padding-bottom:calc(10px + env(safe-area-inset-bottom)); background:#0b1526; } .create-issue-capture-actions[hidden] { display:none; } .create-issue-filing { display:grid; gap:12px; min-width:0; } diff --git a/frontend/dashboard.js b/frontend/dashboard.js index 0f9c130..a1c509c 100644 --- a/frontend/dashboard.js +++ b/frontend/dashboard.js @@ -735,12 +735,23 @@ }); let sharedLaunchState = null; let sharedLaunchHandled = false; + let voiceIssueCapture = null; async function ensureIssueCapture() { return await issueCaptureFeatures.run('issue-capture', { trigger: qs('#new-issue'), status: qs('#my-work-action-status'), }, () => { if (!issueCapture) { issueCapture = createIssueCapture({ fetchJson: fetchReviewJson, storage: localStorage }); + voiceIssueCapture = createVoiceIssueCapture({ + Recognition: window.SpeechRecognition || window.webkitSpeechRecognition, + elements: { + root:qs('#voice-issue-capture'), start:qs('#start-voice-issue-capture'), + stop:qs('#stop-voice-issue-capture'), review:qs('#voice-issue-review'), + transcript:qs('#voice-issue-transcript'), append:qs('#append-voice-issue-transcript'), + replace:qs('#replace-with-voice-issue-transcript'), status:qs('#voice-issue-status'), + title:qs('#create-issue-title'), body:qs('#create-issue-body'), + }, + }); issueOwnerPicker = createIssueCapture.createOwnerPicker(issueCapture, document, () => { saveIssueCaptureDraft(); updateIssueCreateActions(); }); issueTemplatePicker = issueCapture.bindTemplatePicker({ @@ -4343,6 +4354,7 @@ return; } qs('#create-issue-sheet').classList.remove('open'); + voiceIssueCapture.cancel(); mobileComposerViewport.close(qs('.create-issue-panel')); clearTimeout(duplicateCheckTimer); qs('#create-issue-duplicates').hidden = true; diff --git a/frontend/index.html b/frontend/index.html index bf33e6d..d29c810 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -816,6 +816,26 @@ +
@@ -1349,6 +1369,7 @@ + diff --git a/frontend/service-worker.js b/frontend/service-worker.js index 18a21c8..605c939 100644 --- a/frontend/service-worker.js +++ b/frontend/service-worker.js @@ -75,6 +75,7 @@ const SHELL = [ BASE + 'static/issue-filing-review.js', BASE + 'static/issue-sheet.js', BASE + 'static/checklist-conflict.js', + BASE + 'static/voice-issue-capture.js', BASE + 'static/create-issue-sheet.js', BASE + 'static/create-and-start.js', BASE + 'static/assign-and-start.js', diff --git a/frontend/voice-issue-capture.js b/frontend/voice-issue-capture.js new file mode 100644 index 0000000..ac80384 --- /dev/null +++ b/frontend/voice-issue-capture.js @@ -0,0 +1,83 @@ +function mapVoiceTranscript(value) { + const transcript = String(value || '').replace(/\s+/g, ' ').trim(); + if (!transcript) return {title:'', body:''}; + const sentence = transcript.match(/^(.+?[.!?])(?:\s+|$)(.*)$/); + const title = (sentence ? sentence[1] : transcript).trim().slice(0, 255); + const body = (sentence ? sentence[2] : '').trim().slice(0, 10000); + return {title, body}; +} + +function createVoiceIssueCapture({Recognition, elements, createEvent = name => new Event(name, {bubbles:true})}) { + const supported = typeof Recognition === 'function'; + elements.root.hidden = !supported; + if (!supported) { + elements.status.textContent = 'Voice capture is unavailable; type the title and note instead.'; + return {supported:false, cancel() {}}; + } + let recognition = null; + + function showReview(value) { + elements.transcript.value = value; + const hasDraft = Boolean(elements.title.value.trim() || elements.body.value.trim()); + elements.append.textContent = hasDraft ? 'Append to draft' : 'Use transcript'; + elements.replace.hidden = !hasDraft; + elements.review.hidden = false; + elements.status.textContent = 'Transcript ready. Review it before using it.'; + } + + elements.start.addEventListener('click', () => { + recognition = new Recognition(); + recognition.continuous = true; + recognition.interimResults = false; + recognition.onresult = event => { + const finalText = Array.from(event.results || []) + .filter(result => result.isFinal) + .map(result => result[0]?.transcript || '').join(' ').replace(/\s+/g, ' ').trim(); + if (finalText) showReview(finalText); + }; + recognition.onerror = event => { + elements.start.hidden = false; + elements.stop.hidden = true; + elements.status.textContent = event.error === 'not-allowed' || event.error === 'service-not-allowed' ? + 'Microphone permission was denied. Your draft and transcript are unchanged.' : + 'Voice capture stopped. Your draft and transcript are unchanged.'; + }; + recognition.onend = () => { + elements.start.hidden = false; + elements.stop.hidden = true; + }; + recognition.start(); + elements.start.hidden = true; + elements.stop.hidden = false; + elements.status.textContent = 'Listening… Tap stop when you are finished.'; + }); + elements.stop.addEventListener('click', () => { + recognition?.stop(); + elements.status.textContent = 'Finishing transcript…'; + }); + + function commit(mode) { + const mapped = mapVoiceTranscript(elements.transcript.value); + if (mode === 'replace') { + elements.title.value = mapped.title; + elements.body.value = mapped.body; + } else if (elements.title.value.trim()) { + elements.body.value = [elements.body.value.trim(), mapped.title, mapped.body] + .filter(Boolean).join('\n\n').slice(0, 10000); + } else { + elements.title.value = mapped.title; + elements.body.value = [elements.body.value.trim(), mapped.body] + .filter(Boolean).join('\n\n').slice(0, 10000); + } + elements.title.dispatchEvent(createEvent('input')); + elements.body.dispatchEvent(createEvent('input')); + elements.review.hidden = true; + elements.status.textContent = 'Transcript added. Review the title and note before saving.'; + } + elements.append.addEventListener('click', () => commit('append')); + elements.replace.addEventListener('click', () => commit('replace')); + + return {supported:true, cancel() { recognition?.abort(); recognition = null; }}; +} + +if (typeof module !== 'undefined') module.exports = {mapVoiceTranscript, createVoiceIssueCapture}; diff --git a/src/frontend_bundle.py b/src/frontend_bundle.py index ac5fb0e..322cdde 100644 --- a/src/frontend_bundle.py +++ b/src/frontend_bundle.py @@ -23,7 +23,7 @@ WORKER_RUNTIME_SOURCE = "static/background-issue-sync.js" FEATURE_SOURCES = { "comment-actions": ("static/conversation.js", "static/comment-actions.js"), "issue-capture": ( - "static/create-issue-sheet.js", "static/update-follow-up.js", "static/shared-image-capture.js", + "static/voice-issue-capture.js", "static/create-issue-sheet.js", "static/update-follow-up.js", "static/shared-image-capture.js", ), "pull-workflow": ("static/pull-sheet.js", "static/review-sheet.js"), "push-notifications": ("static/push-notifications.js",), diff --git a/tests/e2e/test_mobile_offline_issue_release.py b/tests/e2e/test_mobile_offline_issue_release.py index b7f4754..799b5d0 100644 --- a/tests/e2e/test_mobile_offline_issue_release.py +++ b/tests/e2e/test_mobile_offline_issue_release.py @@ -24,7 +24,7 @@ from fake_gitea import FakeGiteaServer ROOT = Path(__file__).resolve().parents[2] -TITLE = "Offline artifact journey 863" +TITLE = "Offline artifact journey 863." BODY = "Captured on a phone, retained offline, delivered exactly once." ACCESS_TOKEN = "artifact-browser-access-token-863" @@ -148,6 +148,14 @@ def test_release_artifact_files_one_mobile_issue_exactly_once_after_offline_relo service_workers="allow", ignore_https_errors=True, ) + context.add_init_script(""" + class DeterministicSpeechRecognition { + start() { window.__voiceRecognition = this; } + stop() { if (this.onend) this.onend(); } + abort() { if (this.onend) this.onend(); } + } + window.SpeechRecognition = DeterministicSpeechRecognition; + """) page = context.new_page() page.on("pageerror", lambda error: browser_errors.append(error.stack or str(error))) page.on("console", lambda message: browser_errors.append(message.text) if message.type == "error" else None) @@ -178,8 +186,18 @@ def test_release_artifact_files_one_mobile_issue_exactly_once_after_offline_relo assert page.evaluate("() => navigator.serviceWorker.controller !== null") new_action.click() - page.locator("#create-issue-title").fill(TITLE) - page.locator("#create-issue-body").fill(BODY) + expect(page.locator("#voice-issue-capture")).to_be_visible() + page.locator("#start-voice-issue-capture").click() + page.evaluate("""([title, body]) => { + const result = [{ transcript: title + ' ' + body }]; + result.isFinal = true; + window.__voiceRecognition.onresult({ results: [result] }); + }""", [TITLE, BODY]) + expect(page.locator("#voice-issue-review")).to_be_visible() + expect(page.locator("#append-voice-issue-transcript")).to_have_text("Use transcript") + page.locator("#append-voice-issue-transcript").click() + expect(page.locator("#create-issue-title")).to_have_value(TITLE) + expect(page.locator("#create-issue-body")).to_have_value(BODY) page.locator("#file-new-issue").click() expect(page.locator("#create-issue-filing")).to_be_visible() page.locator("#create-issue-repository").select_option("acme/mobile") diff --git a/tests/test_service_worker.py b/tests/test_service_worker.py index 68342ab..c2879e6 100644 --- a/tests/test_service_worker.py +++ b/tests/test_service_worker.py @@ -973,6 +973,7 @@ def test_install_precaches_complete_subpath_scoped_app_shell(): "/dashboard/static/issue-filing-review.js", "/dashboard/static/issue-sheet.js", "/dashboard/static/checklist-conflict.js", + "/dashboard/static/voice-issue-capture.js", "/dashboard/static/create-issue-sheet.js", "/dashboard/static/create-and-start.js", "/dashboard/static/assign-and-start.js", diff --git a/tests/test_voice_issue_capture.py b/tests/test_voice_issue_capture.py new file mode 100644 index 0000000..cdaeaa9 --- /dev/null +++ b/tests/test_voice_issue_capture.py @@ -0,0 +1,200 @@ +import json +import subprocess +from pathlib import Path + +from src.frontend_bundle import build_frontend + + +VOICE_CAPTURE = Path(__file__).parents[1] / "frontend" / "voice-issue-capture.js" +FRONTEND = VOICE_CAPTURE.parent + + +def run_node(script: str): + completed = subprocess.run( + ["node", "-e", script], capture_output=True, text=True + ) + assert completed.returncode == 0, completed.stderr + return json.loads(completed.stdout) + + +def test_transcript_maps_first_sentence_to_title_and_remainder_to_note_with_field_limits(): + script = f""" +const {{mapVoiceTranscript}} = require({json.dumps(str(VOICE_CAPTURE))}); +const mapped = mapVoiceTranscript(' Lift outage on level four. Elevator B is trapped. '); +const bounded = mapVoiceTranscript('x'.repeat(300) + '. ' + 'y'.repeat(10100)); +process.stdout.write(JSON.stringify({{mapped, titleLength:bounded.title.length, bodyLength:bounded.body.length}})); +""" + assert run_node(script) == { + "mapped": { + "title": "Lift outage on level four.", + "body": "Elevator B is trapped.", + }, + "titleLength": 255, + "bodyLength": 10000, + } + + +def test_explicit_tap_starts_recognition_and_final_transcript_waits_for_review(): + script = f""" +const {{createVoiceIssueCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +class Element {{ + constructor() {{ this.hidden=false; this.value=''; this.textContent=''; this.listeners={{}}; }} + addEventListener(name, callback) {{ this.listeners[name]=callback; }} + click() {{ this.listeners.click?.({{currentTarget:this}}); }} + dispatchEvent() {{}} +}} +let instance; +class Recognition {{ + constructor() {{ instance=this; this.started=0; }} + start() {{ this.started += 1; }} + stop() {{}} + abort() {{}} +}} +const elements = Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','status','title','body'].map(key=>[key,new Element()]) +); +elements.review.hidden=true; +const controller = createVoiceIssueCapture({{Recognition,elements}}); +const before = {{started:Boolean(instance), rootHidden:elements.root.hidden}}; +elements.start.click(); +instance.onresult({{results:[Object.assign([{{transcript:'Lift outage. Elevator B is trapped.'}}], {{isFinal:true}})]}}); +process.stdout.write(JSON.stringify({{ + supported:controller.supported, before, started:instance.started, + listening:elements.status.textContent, transcript:elements.transcript.value, + reviewHidden:elements.review.hidden, title:elements.title.value, body:elements.body.value, + appendLabel:elements.append.textContent, replaceHidden:elements.replace.hidden +}})); +""" + assert run_node(script) == { + "supported": True, + "before": {"started": False, "rootHidden": False}, + "started": 1, + "listening": "Transcript ready. Review it before using it.", + "transcript": "Lift outage. Elevator B is trapped.", + "reviewHidden": False, + "title": "", + "body": "", + "appendLabel": "Use transcript", + "replaceHidden": True, + } + + +def test_review_requires_explicit_append_or_replace_when_typed_content_exists(): + script = f""" +const {{createVoiceIssueCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +class Element {{ + constructor() {{ this.hidden=false; this.value=''; this.textContent=''; this.listeners={{}}; this.events=[]; }} + addEventListener(name, callback) {{ this.listeners[name]=callback; }} + click() {{ this.listeners.click?.({{currentTarget:this}}); }} + dispatchEvent(event) {{ this.events.push(event.type); }} +}} +function setup() {{ + let instance; + class Recognition {{ + constructor() {{ instance=this; }} start() {{}} stop() {{}} abort() {{}} + }} + const elements=Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','status','title','body'].map(key=>[key,new Element()]) + ); + elements.title.value='Existing title'; elements.body.value='Existing note'; elements.review.hidden=true; + createVoiceIssueCapture({{Recognition,elements, createEvent:name=>({{type:name}})}}); + elements.start.click(); + instance.onresult({{results:[Object.assign([{{transcript:'Lift outage. Elevator B is trapped.'}}], {{isFinal:true}})]}}); + return elements; +}} +const appended=setup(); appended.append.click(); +const replaced=setup(); replaced.replace.click(); +process.stdout.write(JSON.stringify({{ + appended:{{title:appended.title.value,body:appended.body.value,events:[appended.title.events,appended.body.events]}}, + replaced:{{title:replaced.title.value,body:replaced.body.value,events:[replaced.title.events,replaced.body.events]}} +}})); +""" + assert run_node(script) == { + "appended": { + "title": "Existing title", + "body": "Existing note\n\nLift outage.\n\nElevator B is trapped.", + "events": [["input"], ["input"]], + }, + "replaced": { + "title": "Lift outage.", + "body": "Elevator B is trapped.", + "events": [["input"], ["input"]], + }, + } + + +def test_unsupported_browser_keeps_keyboard_flow_and_exposes_no_voice_control(): + script = f""" +const {{createVoiceIssueCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +const element=()=>({{hidden:false,textContent:'',addEventListener(){{}}}}); +const elements=Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','status','title','body'].map(key=>[key,element()]) +); +const controller=createVoiceIssueCapture({{Recognition:null,elements}}); +process.stdout.write(JSON.stringify({{ + supported:controller.supported, hidden:elements.root.hidden, guidance:elements.status.textContent +}})); +""" + assert run_node(script) == { + "supported": False, + "hidden": True, + "guidance": "Voice capture is unavailable; type the title and note instead.", + } + + +def test_stop_error_and_sheet_close_release_recognition_without_losing_transcript(): + script = f""" +const {{createVoiceIssueCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +class Element {{ + constructor() {{ this.hidden=false; this.value=''; this.textContent=''; this.listeners={{}}; }} + addEventListener(name, callback) {{ this.listeners[name]=callback; }} + click() {{ this.listeners.click?.({{currentTarget:this}}); }} dispatchEvent() {{}} +}} +let instance; +class Recognition {{ + constructor() {{ instance=this; this.stops=0; this.aborts=0; }} + start() {{}} stop() {{ this.stops+=1; }} abort() {{ this.aborts+=1; }} +}} +const elements=Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','status','title','body'].map(key=>[key,new Element()]) +); +const controller=createVoiceIssueCapture({{Recognition,elements}}); +elements.start.click(); +instance.onresult({{results:[Object.assign([{{transcript:'Partial final transcript.'}}], {{isFinal:true}})]}}); +elements.stop.click(); +const afterStop=elements.status.textContent; +instance.onerror({{error:'not-allowed'}}); +const afterError=elements.status.textContent; +controller.cancel(); +process.stdout.write(JSON.stringify({{ + stops:instance.stops, aborts:instance.aborts, afterStop, afterError, + transcript:elements.transcript.value, reviewHidden:elements.review.hidden +}})); +""" + assert run_node(script) == { + "stops": 1, + "aborts": 1, + "afterStop": "Finishing transcript…", + "afterError": "Microphone permission was denied. Your draft and transcript are unchanged.", + "transcript": "Partial final transcript.", + "reviewHidden": False, + } + + +def test_mobile_new_sheet_ships_accessible_voice_review_inside_issue_capture_feature(): + build = build_frontend(FRONTEND) + html = build.dashboard_html + capture_bundle = build.feature_bundles["issue-capture"].runtime_bytes + dashboard = (FRONTEND / "dashboard.js").read_text() + + assert 'id="voice-issue-capture"' in html + assert 'id="start-voice-issue-capture"' in html + assert 'id="stop-voice-issue-capture"' in html + assert 'id="voice-issue-transcript"' in html + assert 'id="voice-issue-status" class="small" aria-live="polite"' in html + assert 'id="append-voice-issue-transcript"' in html + assert 'id="replace-with-voice-issue-transcript"' in html + assert b"function createVoiceIssueCapture" in capture_bundle + assert b"function createVoiceIssueCapture" not in build.runtime_bytes + assert "window.SpeechRecognition || window.webkitSpeechRecognition" in dashboard + assert "voiceIssueCapture.cancel();" in dashboard