Merge main into ios-port and update the iOS CFW transport

This commit is contained in:
James Babcock committed 2026-09-17 23:33:51 -07:00
commit e8e144f3f2
156 files changed
+10528 -1679

No files matched your search

+70 -20
View File
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
// After native recognition finishes, a cloud final can still trail in.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
@@ -64,7 +63,11 @@ export type VoiceInputLayerOptions = {
*/
export class VoiceInputLayer implements Layer {
private phase: VoicePhase = "capturing";
private status = "Listening...";
private status = "Starting microphone...";
/** Mirrors the bridge: only true while audio is actually being transcribed. */
private listening = false;
/** Bridge guidance shown in place of the empty transcript (e.g. "check your phone"). */
private detail = "";
// The active utterance. displayText() is what the dialog shows and what
// Send delivers; the refine flow also writes the merged result here.
private finalizedText = "";
@@ -76,6 +79,10 @@ export class VoiceInputLayer implements Layer {
private capturing = false;
/** Continuation stopped; waiting for the trailing final transcript. */
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private pendingFollowup = false;
private stoppingCapture = false;
private stopGeneration = 0;
private acceptingTranscript = true;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
@@ -114,6 +121,8 @@ export class VoiceInputLayer implements Layer {
// The refine stage owns the status line ("Refining...", error text).
if (this.phase === "refining") return;
this.status = state.status;
this.listening = state.listening;
this.detail = state.detail;
this.actions.requestRender();
});
if (this.handsFree || global.isIOS) {
@@ -149,19 +158,48 @@ export class VoiceInputLayer implements Layer {
}
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.startsWith("Listening")) {
} else if (this.status.endsWith("...")) {
// A progress status ("Starting microphone...", "Listening...") gives
// way to the menu prompt; an error (ending in ".") stays visible.
this.status = "Send, continue, or discard?";
}
this.stopCapture(() => {
if (this.pendingAutoSend) {
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
}
/** Native stop can outlast its synchronous join while Whisper is decoding. */
private stopCapture(onStopped: () => void): void {
const generation = ++this.stopGeneration;
this.stoppingCapture = true;
const afterStop = () => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
onStopped();
this.actions.requestRender();
};
const stopped = this.actions.stopVoiceCapture();
if (!stopped) {
afterStop();
return;
}
void stopped.then(afterStop, (error) => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
this.pendingAutoSend = false;
this.acceptingTranscript = false;
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
});
}
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
@@ -185,7 +223,7 @@ export class VoiceInputLayer implements Layer {
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasText = text.length > 0 && !this.stoppingCapture;
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
@@ -193,6 +231,7 @@ export class VoiceInputLayer implements Layer {
label: target.label,
dim: !hasText,
onSelect: () => {
if (this.stoppingCapture) return;
this.dismiss();
if (hasText) target.onSend(text);
},
@@ -200,9 +239,9 @@ export class VoiceInputLayer implements Layer {
}
rows.push({
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
dim: !hasLlmKey || this.stoppingCapture,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
@@ -213,8 +252,8 @@ export class VoiceInputLayer implements Layer {
const image = paintBelow();
const inMenu = this.phase === "menu";
paintInputDialog(image, {
title: this.capturing ? "Voice ●" : "Voice",
status: this.status,
title: this.capturing && this.listening ? "Voice ●" : "Voice",
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
text: this.displayText() || this.placeholderText(),
rows: inMenu ? this.menuRows() : [],
selectedRow: this.menuIndex,
@@ -276,6 +315,7 @@ export class VoiceInputLayer implements Layer {
/** Continue selected: keep the message aside and record a follow-up. */
private startContinuation(): void {
this.acceptingTranscript = true;
this.baseText = this.displayText().trim();
this.finalizedText = "";
this.liveText = "";
@@ -290,15 +330,18 @@ export class VoiceInputLayer implements Layer {
private endContinuationCapture(): void {
if (!this.capturing) return;
this.capturing = false;
void this.actions.stopVoiceCapture();
this.status = "Refining...";
this.pendingFollowup = true;
this.status = "Finishing transcription...";
this.stopCapture(() => {
if (this.pendingFollowup) {
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
// The provider's committed transcript arrives shortly after stop; refine
// when it does, or after a timeout with whatever partials we have.
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
private beginRefine(): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -335,9 +378,10 @@ export class VoiceInputLayer implements Layer {
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
private cancelContinuation(status: string): void {
this.acceptingTranscript = false;
if (this.capturing) {
this.capturing = false;
void this.actions.stopVoiceCapture();
this.stopCapture(() => {});
}
this.refineHandle?.cancel();
this.refineHandle = null;
@@ -345,6 +389,7 @@ export class VoiceInputLayer implements Layer {
}
private backToMenu(text: string, status: string): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -358,6 +403,9 @@ export class VoiceInputLayer implements Layer {
}
onRemoved(): void {
++this.stopGeneration;
this.acceptingTranscript = false;
this.pendingFollowup = false;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
@@ -392,7 +440,9 @@ export class VoiceInputLayer implements Layer {
private placeholderText(): string {
switch (this.phase) {
case "capturing":
return "Listening...";
// Only claim to listen when the mic is actually running; otherwise
// the bridge's guidance (permission prompt on the phone, etc.), if any.
return this.listening ? "Listening..." : this.detail;
case "continuing":
return "Say more, or describe an edit...";
case "refining":
@@ -419,14 +469,14 @@ export class VoiceInputLayer implements Layer {
private onTranscript(event: VoiceTranscriptEvent): void {
// The refine stream owns the text buffers once it starts; a transcript
// that trails in after that point is stale.
if (this.phase === "refining") return;
if (!this.acceptingTranscript || this.phase === "refining") return;
if (event.isFinal) {
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
this.liveText = "";
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
if (this.phase === "continuing" && this.pendingFollowup) {
// The follow-up finalized; no need to keep waiting.
this.beginRefine();
return;