From 0e13872311660cf38a5e033f9c7a27c456989389 Mon Sep 17 00:00:00 2001 From: timmy Date: Sun, 16 Aug 2026 08:35:34 +0000 Subject: [PATCH] feat: dictate mobile conversation replies (Closes #951) --- frontend/dashboard.css | 6 + frontend/dashboard.js | 30 ++++ frontend/index.html | 49 ++++++ frontend/service-worker.js | 1 + frontend/voice-conversation-capture.js | 124 +++++++++++++++ frontend/voice-transcript-store.js | 18 ++- src/frontend_bundle.py | 1 + tests/test_service_worker.py | 1 + tests/test_voice_conversation_capture.py | 187 +++++++++++++++++++++++ 9 files changed, 411 insertions(+), 6 deletions(-) create mode 100644 frontend/voice-conversation-capture.js create mode 100644 tests/test_voice_conversation_capture.py diff --git a/frontend/dashboard.css b/frontend/dashboard.css index 7cf33a7..4f7d4c9 100644 --- a/frontend/dashboard.css +++ b/frontend/dashboard.css @@ -681,6 +681,12 @@ textarea { resize: vertical; min-height: 120px; } .voice-issue-controls, .voice-issue-review-actions { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:8px; } .voice-issue-controls button, .voice-issue-review-actions button { min-width:0; min-height:44px; } #voice-issue-transcript { box-sizing:border-box; width:100%; min-height:96px; resize:vertical; } +.voice-conversation { display:grid; gap:8px; min-width:0; padding:10px; border:1px solid #31577f; border-radius:12px; background:#101d31; } +.voice-conversation[hidden], .voice-conversation-review[hidden], .voice-conversation-controls button[hidden] { display:none; } +.voice-conversation-controls, .voice-conversation-review-actions { display:grid; grid-template-columns:repeat(auto-fit,minmax(120px,1fr)); gap:8px; } +.voice-conversation-controls button, .voice-conversation-review-actions button { min-height:44px; min-width:0; } +.voice-conversation-review { display:grid; gap:6px; } +.voice-conversation-review textarea { box-sizing:border-box; width:100%; min-height:96px; resize:vertical; } .create-issue-capture-actions { position:sticky; bottom:0; display:grid; grid-template-columns:1fr 1fr; gap:8px; padding:10px 0; padding-bottom:calc(10px + env(safe-area-inset-bottom)); background:#0b1526; } .create-issue-capture-actions[hidden] { display:none; } .create-issue-filing { display:grid; gap:12px; min-width:0; } diff --git a/frontend/dashboard.js b/frontend/dashboard.js index 0b256d3..90cb3c6 100644 --- a/frontend/dashboard.js +++ b/frontend/dashboard.js @@ -622,6 +622,27 @@ body:issueAttachment.multipart(payload) }, ), }); + const voiceTranscriptStore = createVoiceTranscriptStore(); + function mountConversationVoice(kind, draftSelector) { + return createVoiceConversationCapture({ + Recognition: window.SpeechRecognition || window.webkitSpeechRecognition, + transcriptStore: voiceTranscriptStore, + getLogin: () => confirmedOwnerLogin, + elements: { + root:qs('#voice-' + kind), start:qs('#start-voice-' + kind), + stop:qs('#stop-voice-' + kind), review:qs('#voice-' + kind + '-review'), + transcript:qs('#voice-' + kind + '-transcript'), append:qs('#append-voice-' + kind), + replace:qs('#replace-with-voice-' + kind), discard:qs('#discard-voice-' + kind), + status:qs('#voice-' + kind + '-status'), draft:qs(draftSelector), + }, + }); + } + const issueVoiceReply = mountConversationVoice('issue-comment', '#issue-comment'); + const pullVoiceReply = mountConversationVoice('pull-comment', '#pull-comment'); + const updateVoiceReply = mountConversationVoice('update-reply', '#update-reply'); + function conversationVoiceTarget(kind, item) { + return kind + ':' + String(item?.repository || '').trim().toLowerCase() + '#' + Number(item?.number); + } const createIssueAttachmentController = issueAttachment.mount({ input: qs('#create-issue-attachment'), inputs: [qs('#take-create-issue-photo'), qs('#create-issue-attachment')], @@ -1218,6 +1239,7 @@ updateReplyAttachmentController.clear(); } selectedUpdate = item; + void updateVoiceReply.open('update:' + item.notification_id); selectedUpdateDetail = null; updateReadPosition.open(String(item.notification_id)); updateMentions.dismiss(); @@ -1470,6 +1492,9 @@ qs('#retry-work-route').addEventListener('click', () => workRoute.sync()); function closeOpenWorkSheets() { + issueVoiceReply.cancel(); + pullVoiceReply.cancel(); + updateVoiceReply.cancel(); ['#issue-sheet .issue-sheet-panel', '#pull-sheet .pull-sheet-panel', '#review-sheet .review-sheet-panel', '#update-sheet .update-sheet-panel'] .forEach(selector => mobileComposerViewport.close(qs(selector))); ['#issue-sheet', '#pull-sheet', '#review-sheet', '#update-sheet'].forEach(selector => @@ -3574,6 +3599,7 @@ qs('#issue-planning').inert = false; qs('#issue-handoff').inert = false; selectedIssue = item; + void issueVoiceReply.open(conversationVoiceTarget('issue', item)); dismissedChecklistBody = null; issueMentions.dismiss(); selectedIssueOffline = Boolean(offlineDetail); @@ -3739,6 +3765,7 @@ return; } mobileComposerViewport.close(qs('#issue-sheet .issue-sheet-panel')); + issueVoiceReply.cancel(); issueAttachmentController.clear(); qs('#issue-sheet').classList.remove('open'); selectedIssue = null; @@ -3895,6 +3922,7 @@ qs('#pull-review').inert = false; qs('#pull-ownership').inert = false; selectedPull = item; + void pullVoiceReply.open(conversationVoiceTarget('pull', item)); pullMentions.dismiss(); pullTrigger = trigger; selectedPullDetail = null; @@ -3959,6 +3987,7 @@ return; } mobileComposerViewport.close(qs('#pull-sheet .pull-sheet-panel')); + pullVoiceReply.cancel(); pullAttachmentController.clear(); qs('#pull-sheet').classList.remove('open'); selectedPull = null; @@ -4688,6 +4717,7 @@ return; } mobileComposerViewport.close(qs('#update-sheet .update-sheet-panel')); + updateVoiceReply.cancel(); updateReplyAttachmentController.clear(); qs('#update-sheet').classList.remove('open'); selectedUpdate = null; diff --git a/frontend/index.html b/frontend/index.html index d59c808..96d5139 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -659,6 +659,22 @@

Add comment

+
@@ -1060,6 +1076,22 @@

Reply

+
@@ -1142,6 +1174,22 @@

Add comment

+
@@ -1410,6 +1458,7 @@ + diff --git a/frontend/service-worker.js b/frontend/service-worker.js index 6ce72dd..b053094 100644 --- a/frontend/service-worker.js +++ b/frontend/service-worker.js @@ -77,6 +77,7 @@ const SHELL = [ BASE + 'static/checklist-conflict.js', BASE + 'static/voice-transcript-store.js', BASE + 'static/voice-issue-capture.js', + BASE + 'static/voice-conversation-capture.js', BASE + 'static/create-issue-sheet.js', BASE + 'static/create-and-start.js', BASE + 'static/assign-and-start.js', diff --git a/frontend/voice-conversation-capture.js b/frontend/voice-conversation-capture.js new file mode 100644 index 0000000..8b04690 --- /dev/null +++ b/frontend/voice-conversation-capture.js @@ -0,0 +1,124 @@ +function createVoiceConversationCapture({ + Recognition, elements, transcriptStore = null, getLogin = () => '', + createEvent = name => new Event(name, {bubbles:true}), +}) { + const supported = typeof Recognition === 'function'; + let recognition = null; + let target = ''; + let generation = 0; + elements.root.hidden = !supported; + if (!supported) { + elements.status.textContent = 'Voice capture is unavailable; type the reply instead.'; + } + + const normalize = value => String(value || '').replace(/\s+/g, ' ').trim(); + const login = () => String(getLogin() || '').trim(); + + function showReview(value, recovered = false) { + const transcript = normalize(value).slice(0, 10000); + elements.transcript.value = transcript; + elements.review.hidden = false; + elements.append.textContent = elements.draft.value.trim() ? 'Append to draft' : 'Use transcript'; + elements.replace.hidden = !elements.draft.value.trim(); + elements.status.textContent = recovered ? + 'Recovered transcript. Review it before using or discarding it.' : + 'Transcript ready. Review it before using it.'; + if (!recovered && transcriptStore && login() && target) { + void transcriptStore.save(login(), transcript, target); + } + } + + async function open(nextTarget) { + cancel(); + target = String(nextTarget || '').trim(); + const opening = ++generation; + elements.transcript.value = ''; + elements.review.hidden = true; + elements.status.textContent = ''; + if (!supported || !target || !transcriptStore || !login()) return; + const recovered = normalize(await transcriptStore.load(login(), target)).slice(0, 10000); + if (opening === generation && recovered) showReview(recovered, true); + } + + function clearCheckpoint() { + if (transcriptStore && login() && target) void transcriptStore.clear(login(), target); + } + + function commit(replace) { + const transcript = normalize(elements.transcript.value).slice(0, 10000); + elements.draft.value = replace ? transcript : + [elements.draft.value.trim(), transcript].filter(Boolean).join('\n\n').slice(0, 10000); + elements.draft.dispatchEvent(createEvent('input')); + clearCheckpoint(); + elements.review.hidden = true; + elements.status.textContent = 'Transcript added. Review the reply before sending.'; + } + + elements.start.addEventListener('click', () => { + if (!supported || !target) return; + const sessionGeneration = generation; + const currentRecognition = new Recognition(); + recognition = currentRecognition; + recognition.continuous = true; + recognition.interimResults = false; + recognition.onresult = event => { + if (recognition !== currentRecognition || generation !== sessionGeneration) return; + const finalText = Array.from(event.results || []).slice(event.resultIndex ?? 0) + .filter(result => result.isFinal) + .map(result => result[0]?.transcript || '').join(' '); + if (normalize(finalText)) { + showReview([elements.transcript.value, finalText].filter(Boolean).join(' ')); + } + }; + recognition.onerror = event => { + if (recognition !== currentRecognition || generation !== sessionGeneration) return; + elements.start.hidden = false; + elements.stop.hidden = true; + elements.status.textContent = ['not-allowed', 'service-not-allowed'].includes(event.error) ? + 'Microphone permission was denied. Your draft and transcript are unchanged.' : + 'Voice capture stopped. Your draft and transcript are unchanged.'; + }; + recognition.onend = () => { + if (recognition !== currentRecognition || generation !== sessionGeneration) return; + recognition = null; + elements.start.hidden = false; + elements.stop.hidden = true; + }; + try { + recognition.start(); + } catch (_) { + recognition = null; + elements.start.hidden = false; + elements.stop.hidden = true; + elements.status.textContent = 'Voice capture could not start. Your draft and transcript are unchanged.'; + return; + } + elements.start.hidden = true; + elements.stop.hidden = false; + elements.status.textContent = 'Listening… Tap stop when you are finished.'; + }); + elements.stop.addEventListener('click', () => { + recognition?.stop(); + elements.status.textContent = 'Finishing transcript…'; + }); + elements.append.addEventListener('click', () => commit(false)); + elements.replace.addEventListener('click', () => commit(true)); + elements.discard.addEventListener('click', () => { + clearCheckpoint(); + elements.transcript.value = ''; + elements.review.hidden = true; + elements.status.textContent = 'Transcript discarded.'; + }); + + function cancel() { + generation += 1; + recognition?.abort(); + recognition = null; + elements.start.hidden = false; + elements.stop.hidden = true; + } + + return {supported, open, cancel}; +} + +if (typeof module !== 'undefined') module.exports = {createVoiceConversationCapture}; diff --git a/frontend/voice-transcript-store.js b/frontend/voice-transcript-store.js index 1ad4cc6..dda114c 100644 --- a/frontend/voice-transcript-store.js +++ b/frontend/voice-transcript-store.js @@ -58,23 +58,29 @@ return String(value || '').replace(/\s+/g, ' ').trim().slice(0, MAX_TRANSCRIPT_LENGTH); } + function storageKey(login, target = '') { + const account = normalizeLogin(login); + const scope = String(target || '').trim().toLowerCase(); + return account && scope ? account + '\u0000' + scope : account; + } + function createVoiceTranscriptStore({transaction = createIndexedDbTransaction(defaultIndexedDB)} = {}) { - async function load(login) { - const key = normalizeLogin(login); + async function load(login, target = '') { + const key = storageKey(login, target); if (!key || !transaction) return ''; const record = await transaction(store => store.get(key)); return normalizeTranscript(record?.transcript); } - async function save(login, transcript) { - const key = normalizeLogin(login); + async function save(login, transcript, target = '') { + const key = storageKey(login, target); const value = normalizeTranscript(transcript); if (!key || !value || !transaction) return; await transaction(store => store.put({login:key, transcript:value}), 'readwrite'); } - async function clear(login) { - const key = normalizeLogin(login); + async function clear(login, target = '') { + const key = storageKey(login, target); if (!key || !transaction) return; await transaction(store => store.delete(key), 'readwrite'); } diff --git a/src/frontend_bundle.py b/src/frontend_bundle.py index 260d671..d844bae 100644 --- a/src/frontend_bundle.py +++ b/src/frontend_bundle.py @@ -30,6 +30,7 @@ FEATURE_SOURCES = { "device-setup": ("static/install-app.js", "static/mobile-device-setup.js"), "security-center": ("static/security-center.js",), "today-timer": ( + "static/voice-transcript-store.js", "static/voice-conversation-capture.js", "static/today-completion.js", "static/work-detail-position.js", "static/work-route.js", "static/commands.js", "static/saved-searches.js", "static/task-overlay-history.js", "static/search-preview.js", "static/search-defer.js", "static/mobile-search-viewport.js", "static/my-work.js", "static/protect-today.js", "static/mobile-task-dock.js", "static/mobile-queue-launcher.js", "static/mobile-start-day.js", "static/update-triage-session.js", "static/update-review-handoff.js", "static/update-triage-launcher.js", "static/update-triage-gesture.js", "static/notification-undo.js", "static/today-timer.js", "static/today-recap.js", "static/today-rollover.js", "static/later-work.js", "static/later-picker.js", "static/drafts.js", "static/unfiled-captures.js", "static/unfiled-draft-sync.js", "static/assign-and-start.js", "static/filed-claim.js", "static/queue-today.js", "static/create-and-start.js", diff --git a/tests/test_service_worker.py b/tests/test_service_worker.py index 0684bdd..45f2c9e 100644 --- a/tests/test_service_worker.py +++ b/tests/test_service_worker.py @@ -994,6 +994,7 @@ def test_install_precaches_complete_subpath_scoped_app_shell(): "/dashboard/static/checklist-conflict.js", "/dashboard/static/voice-transcript-store.js", "/dashboard/static/voice-issue-capture.js", + "/dashboard/static/voice-conversation-capture.js", "/dashboard/static/create-issue-sheet.js", "/dashboard/static/create-and-start.js", "/dashboard/static/assign-and-start.js", diff --git a/tests/test_voice_conversation_capture.py b/tests/test_voice_conversation_capture.py new file mode 100644 index 0000000..c122468 --- /dev/null +++ b/tests/test_voice_conversation_capture.py @@ -0,0 +1,187 @@ +import json +import subprocess +from pathlib import Path + + +FRONTEND = Path(__file__).parents[1] / "frontend" +VOICE_CAPTURE = FRONTEND / "voice-conversation-capture.js" +VOICE_STORE = FRONTEND / "voice-transcript-store.js" + + +def run_node(script: str): + completed = subprocess.run(["node", "-e", script], capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr + return json.loads(completed.stdout) + + +def test_dictated_reply_waits_for_review_then_appends_to_the_current_conversation(): + script = f""" +const {{createVoiceConversationCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +class Element {{ + constructor() {{ this.hidden=false; this.value=''; this.textContent=''; this.listeners={{}}; this.events=[]; }} + addEventListener(name, callback) {{ this.listeners[name]=callback; }} + click() {{ this.listeners.click?.(); }} + dispatchEvent(event) {{ this.events.push(event.type); }} +}} +let recognition; +class Recognition {{ + constructor() {{ recognition=this; this.aborts=0; }} + start() {{}} stop() {{}} abort() {{ this.aborts += 1; }} +}} +const elements=Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','discard','status','draft'].map(key=>[key,new Element()]) +); +elements.review.hidden=true; +elements.draft.value='Typed opening'; +const saved=[]; const cleared=[]; +const store={{ + async load(login, target) {{ return ''; }}, + async save(login, transcript, target) {{ saved.push({{login, transcript, target}}); }}, + async clear(login, target) {{ cleared.push({{login, target}}); }}, +}}; +(async()=>{{ + const controller=createVoiceConversationCapture({{ + Recognition,elements,transcriptStore:store,getLogin:()=> 'timmy',createEvent:name=>({{type:name}}) + }}); + await controller.open('issue:stackchain/stackchain-dashboard#951'); + elements.start.click(); + const first=Object.assign([{{transcript:'Please test on mobile.'}}],{{isFinal:true}}); + recognition.onresult({{resultIndex:0,results:[first]}}); + const before={{draft:elements.draft.value,reviewHidden:elements.review.hidden,transcript:elements.transcript.value}}; + elements.append.click(); await Promise.resolve(); + process.stdout.write(JSON.stringify({{ + before,draft:elements.draft.value,events:elements.draft.events,saved,cleared, + reviewHidden:elements.review.hidden,status:elements.status.textContent + }})); +}})().catch(error=>{{console.error(error);process.exit(1);}}); +""" + assert run_node(script) == { + "before": { + "draft": "Typed opening", + "reviewHidden": False, + "transcript": "Please test on mobile.", + }, + "draft": "Typed opening\n\nPlease test on mobile.", + "events": ["input"], + "saved": [ + { + "login": "timmy", + "transcript": "Please test on mobile.", + "target": "issue:stackchain/stackchain-dashboard#951", + } + ], + "cleared": [ + {"login": "timmy", "target": "issue:stackchain/stackchain-dashboard#951"} + ], + "reviewHidden": True, + "status": "Transcript added. Review the reply before sending.", + } + + +def test_private_store_isolates_conversation_transcripts_by_account_and_target(): + script = f""" +const createVoiceTranscriptStore=require({json.dumps(str(VOICE_STORE))}); +const records=new Map(); +const transaction=async work=>work({{ + get:async key=>records.get(key), + put:async value=>records.set(value.login,{{...value}}), + delete:async key=>records.delete(key), +}}); +(async()=>{{ + const store=createVoiceTranscriptStore({{transaction}}); + await store.save('timmy','Issue reply','issue:org/repo#1'); + await store.save('timmy','Pull reply','pull:org/repo#2'); + await store.save('alexander','Other account','issue:org/repo#1'); + const before={{ + issue:await store.load('TIMMY','issue:org/repo#1'), + pull:await store.load('timmy','pull:org/repo#2'), + other:await store.load('alexander','issue:org/repo#1'), + }}; + await store.clear('timmy','issue:org/repo#1'); + process.stdout.write(JSON.stringify({{before,cleared:await store.load('timmy','issue:org/repo#1')}})); +}})().catch(error=>{{console.error(error);process.exit(1);}}); +""" + assert run_node(script) == { + "before": {"issue": "Issue reply", "pull": "Pull reply", "other": "Other account"}, + "cleared": "", + } + + +def test_switching_conversations_aborts_listening_and_recovers_only_the_new_target(): + script = f""" +const {{createVoiceConversationCapture}} = require({json.dumps(str(VOICE_CAPTURE))}); +class Element {{ + constructor() {{ this.hidden=false;this.value='';this.textContent='';this.listeners={{}}; }} + addEventListener(name,callback) {{ this.listeners[name]=callback; }} + click() {{ this.listeners.click?.(); }} dispatchEvent() {{}} +}} +const instances=[]; +class Recognition {{ + constructor() {{ this.aborts=0; instances.push(this); }} start() {{}} stop() {{}} abort() {{ this.aborts+=1; }} +}} +const elements=Object.fromEntries( + ['root','start','stop','review','transcript','append','replace','discard','status','draft'].map(key=>[key,new Element()]) +); +const loads=[]; const saved=[]; +const store={{ + async load(login,target) {{ loads.push(target); return target.endsWith('#2') ? 'Recovered pull reply.' : ''; }}, + async save(login,text,target) {{ saved.push({{text,target}}); }}, async clear() {{}}, +}}; +(async()=>{{ + const controller=createVoiceConversationCapture({{Recognition,elements,transcriptStore:store,getLogin:()=> 'timmy'}}); + await controller.open('issue:org/repo#1'); elements.start.click(); + await controller.open('pull:org/repo#2'); + instances[0].onresult({{results:[Object.assign([{{transcript:'Stale issue words.'}}],{{isFinal:true}})]}}); + instances[0].onerror({{error:'not-allowed'}}); + instances[0].onend(); + await Promise.resolve(); + process.stdout.write(JSON.stringify({{ + loads,saved,aborts:instances[0].aborts,transcript:elements.transcript.value, + reviewHidden:elements.review.hidden,status:elements.status.textContent, + startHidden:elements.start.hidden,stopHidden:elements.stop.hidden + }})); +}})().catch(error=>{{console.error(error);process.exit(1);}}); +""" + assert run_node(script) == { + "loads": ["issue:org/repo#1", "pull:org/repo#2"], + "saved": [], + "aborts": 1, + "transcript": "Recovered pull reply.", + "reviewHidden": False, + "status": "Recovered transcript. Review it before using or discarding it.", + "startHidden": False, + "stopHidden": True, + } + + +def test_all_mobile_conversation_composers_ship_accessible_voice_review_controls(): + html = (FRONTEND / "index.html").read_text() + css = (FRONTEND / "dashboard.css").read_text() + dashboard = (FRONTEND / "dashboard.js").read_text() + bundle = (Path(__file__).parents[1] / "src" / "frontend_bundle.py").read_text() + + for kind, draft in ( + ("issue-comment", "issue-comment"), + ("pull-comment", "pull-comment"), + ("update-reply", "update-reply"), + ): + assert f'id="voice-{kind}"' in html + assert f'id="start-voice-{kind}"' in html + assert f'id="voice-{kind}-transcript"' in html + assert f'for="voice-{kind}-transcript"' in html + assert f'id="append-voice-{kind}"' in html + assert f'id="replace-with-voice-{kind}"' in html + assert f'id="discard-voice-{kind}"' in html + + assert "mountConversationVoice('issue-comment', '#issue-comment')" in dashboard + assert "mountConversationVoice('pull-comment', '#pull-comment')" in dashboard + assert "mountConversationVoice('update-reply', '#update-reply')" in dashboard + assert ".voice-conversation-controls button" in css + assert "min-height:44px" in css + assert '"static/voice-conversation-capture.js"' in bundle + assert "issueVoiceReply.open(conversationVoiceTarget('issue', item))" in dashboard + assert "pullVoiceReply.open(conversationVoiceTarget('pull', item))" in dashboard + assert "updateVoiceReply.open('update:' + item.notification_id)" in dashboard + assert "issueVoiceReply.cancel();" in dashboard + assert "pullVoiceReply.cancel();" in dashboard + assert "updateVoiceReply.cancel();" in dashboard -- 2.43.0