diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseListener.java new file mode 100644 index 00000000..8780d601 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseListener.java @@ -0,0 +1,13 @@ +package com.faceclaw.app; + +/** Callbacks for FaceclawSseRequest, delivered on the constructing thread's Looper. */ +public interface FaceclawSseListener { + /** One line of a successful (2xx) streaming response body, without the newline. */ + void onLine(String line); + /** The server answered with a non-2xx status; body is the full error body. */ + void onHttpError(int code, String body); + /** The 2xx response body ended normally. */ + void onComplete(); + /** Network-level failure (connect, TLS, mid-stream drop). Not called after cancel(). */ + void onFailure(String message); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java new file mode 100644 index 00000000..e6e8e5c5 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java @@ -0,0 +1,144 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; + +/** + * Streaming HTTP POST for the TypeScript side, used for server-sent-events + * APIs (e.g. the Anthropic Messages API with stream=true). The response body + * is delivered line by line as it arrives, so the JS side can parse SSE + * events incrementally. Like FaceclawWebSocket, listener callbacks are posted + * to the Looper of the thread that constructed this object. + */ +public class FaceclawSseRequest { + private static final String TAG = "FaceclawSseRequest"; + private static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); + private static volatile OkHttpClient sharedClient; + + private final Handler callbackHandler; + private final Call call; + private volatile boolean cancelled; + + /** + * Starts the request immediately. headers is a flat alternating + * [name, value, name, value, ...] array (okhttp Headers can't cross the + * JS bridge, and parallel arrays are easy to build with Array.create). + */ + public FaceclawSseRequest(String url, String jsonBody, String[] headers, final FaceclawSseListener listener) { + if (url == null || url.trim().isEmpty()) { + throw new IllegalArgumentException("url is required"); + } + if (listener == null) { + throw new IllegalArgumentException("listener is required"); + } + Looper looper = Looper.myLooper(); + callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); + Request.Builder builder = new Request.Builder() + .url(url.trim()) + .post(RequestBody.create(JSON, jsonBody == null ? "" : jsonBody)); + if (headers != null) { + for (int i = 0; i + 1 < headers.length; i += 2) { + if (headers[i] != null && !headers[i].isEmpty() && headers[i + 1] != null) { + builder.addHeader(headers[i], headers[i + 1]); + } + } + } + call = getClient().newCall(builder.build()); + call.enqueue(new Callback() { + @Override public void onFailure(Call call, IOException e) { + postFailure(listener, String.valueOf(e)); + } + + @Override public void onResponse(Call call, Response response) { + try (ResponseBody body = response.body()) { + if (!response.isSuccessful()) { + final int code = response.code(); + String errorBody = ""; + try { + errorBody = body == null ? "" : body.string(); + } catch (IOException e) { + Log.w(TAG, "error body read failed", e); + } + final String finalBody = errorBody; + post(() -> listener.onHttpError(code, finalBody)); + return; + } + if (body == null) { + post(listener::onComplete); + return; + } + BufferedReader reader = new BufferedReader(body.charStream()); + String line; + while ((line = reader.readLine()) != null) { + final String finalLine = line; + post(() -> listener.onLine(finalLine)); + } + post(listener::onComplete); + } catch (IOException e) { + postFailure(listener, String.valueOf(e)); + } + } + }); + } + + public void cancel() { + cancelled = true; + call.cancel(); + } + + private void post(Runnable runnable) { + if (cancelled) { + return; + } + callbackHandler.post(() -> { + if (cancelled) { + return; + } + try { + runnable.run(); + } catch (Throwable t) { + Log.w(TAG, "listener callback failed", t); + } + }); + } + + private void postFailure(FaceclawSseListener listener, String message) { + if (cancelled) { + return; + } + post(() -> listener.onFailure(message)); + } + + private static OkHttpClient getClient() { + OkHttpClient client = sharedClient; + if (client == null) { + synchronized (FaceclawSseRequest.class) { + if (sharedClient == null) { + // Streaming responses can pause between events (e.g. while + // the model thinks), so the read timeout is generous. The + // Anthropic API sends periodic ping events well within it. + sharedClient = new OkHttpClient.Builder() + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(180, TimeUnit.SECONDS) + .build(); + } + client = sharedClient; + } + } + return client; + } +} diff --git a/app/native/anthropic.ts b/app/native/anthropic.ts new file mode 100644 index 00000000..80fbf2a0 --- /dev/null +++ b/app/native/anthropic.ts @@ -0,0 +1,230 @@ +declare const com: any; + +/** + * Minimal streaming client for the Anthropic Messages API. There is no + * official SDK for the NativeScript runtime, so this speaks raw HTTP/SSE via + * FaceclawSseRequest (okhttp on the Java side), which delivers the response + * body line by line on this isolate's thread. + */ + +const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"; +const ANTHROPIC_VERSION = "2023-06-01"; + +/** Default model for voice continuations (and a sensible general default). */ +export const DEFAULT_LLM_MODEL = "claude-sonnet-5"; + +export type AnthropicMessage = { + role: "user" | "assistant"; + content: string; +}; + +export type AnthropicStreamResult = { + text: string; + /** e.g. "end_turn", "max_tokens", "refusal"; null if the stream never said. */ + stopReason: string | null; +}; + +export type AnthropicStreamOptions = { + apiKey: string; + model?: string; + system?: string; + messages: AnthropicMessage[]; + maxTokens?: number; + /** Thinking/latency tradeoff; voice flows want "low". Default: model default. */ + effort?: "low" | "medium" | "high"; + /** Called with each incremental text delta as it streams in. */ + onTextDelta?: (delta: string, textSoFar: string) => void; + onDone: (result: AnthropicStreamResult) => void; + /** Called at most once, instead of onDone, with a user-displayable message. */ + onError: (message: string) => void; +}; + +export type AnthropicStreamHandle = { + cancel(): void; +}; + +export function streamAnthropicMessage(options: AnthropicStreamOptions): AnthropicStreamHandle { + const apiKey = options.apiKey.trim(); + let request: any = null; + let settled = false; + let text = ""; + let stopReason: string | null = null; + + const fail = (message: string) => { + if (settled) return; + settled = true; + options.onError(message); + }; + const done = () => { + if (settled) return; + settled = true; + options.onDone({ text, stopReason }); + }; + + if (!global.isAndroid) { + // Deferred so the caller gets its handle back before the callback runs. + setTimeout(() => fail("Anthropic API is only wired up on Android"), 0); + return { cancel: () => {} }; + } + if (!apiKey) { + setTimeout(() => fail("No Anthropic API key set"), 0); + return { cancel: () => {} }; + } + + const body: any = { + model: options.model ?? DEFAULT_LLM_MODEL, + max_tokens: options.maxTokens ?? 8192, + stream: true, + messages: options.messages, + }; + if (options.system) body.system = options.system; + if (options.effort) body.output_config = { effort: options.effort }; + + const listener = new com.faceclaw.app.FaceclawSseListener({ + onLine: (line: string) => { + if (settled) return; + const event = parseSseDataLine(String(line)); + if (!event) return; + switch (event.type) { + case "content_block_delta": + if (event.delta?.type === "text_delta") { + const delta = String(event.delta.text ?? ""); + text += delta; + options.onTextDelta?.(delta, text); + } + return; + case "message_delta": + if (event.delta?.stop_reason) { + stopReason = String(event.delta.stop_reason); + } + return; + case "message_stop": + done(); + return; + case "error": + fail(`Anthropic: ${String(event.error?.message ?? event.error?.type ?? "stream error")}`); + return; + default: + // message_start, content_block_start/stop, ping — nothing to do. + return; + } + }, + onHttpError: (code: number, errorBody: string) => { + fail(describeHttpError(Number(code), String(errorBody))); + }, + onComplete: () => { + // Normally message_stop settles first; a clean EOF without it still + // resolves with whatever streamed. + done(); + }, + onFailure: (message: string) => { + fail(`Anthropic connection failed: ${String(message)}`); + }, + }); + + const headers = Array.create("java.lang.String", 4) as string[]; + headers[0] = "x-api-key"; + headers[1] = apiKey; + headers[2] = "anthropic-version"; + headers[3] = ANTHROPIC_VERSION; + try { + request = new com.faceclaw.app.FaceclawSseRequest(ANTHROPIC_URL, JSON.stringify(body), headers, listener); + } catch (error) { + setTimeout(() => fail(`Anthropic request failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => {} }; + } + + return { + cancel: () => { + settled = true; + try { + request?.cancel(); + } catch { + // ignore + } + }, + }; +} + +/** Parse one SSE line; returns the JSON payload of a data: line, else null. */ +function parseSseDataLine(line: string): any | null { + if (!line.startsWith("data:")) return null; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") return null; + try { + return JSON.parse(payload); + } catch { + return null; + } +} + +function describeHttpError(code: number, body: string): string { + let detail = ""; + try { + detail = String(JSON.parse(body)?.error?.message ?? ""); + } catch { + // non-JSON error body; fall through to the generic text + } + switch (code) { + case 401: + return "Anthropic: invalid API key"; + case 403: + return "Anthropic: API key lacks permission"; + case 429: + return "Anthropic: rate limited, try again shortly"; + case 500: + case 529: + return "Anthropic: service overloaded, try again shortly"; + default: + return `Anthropic: HTTP ${code}${detail ? ` (${detail})` : ""}`; + } +} + +const REFINE_SYSTEM_PROMPT = + "You edit dictated text. The user dictated a message, then dictated a follow-up. " + + "If the follow-up is additional content, append it to the message where it naturally fits. " + + "If it describes an edit (a correction, a deletion, or content to insert somewhere specific), apply that edit instead of appending the instruction itself. " + + "Fix only what the follow-up asks; keep the rest of the original wording. " + + "Output only the final text of the message, with no preamble, quotes, or commentary."; + +export type RefineDictationOptions = { + apiKey: string; + original: string; + followup: string; + onTextDelta?: (delta: string, textSoFar: string) => void; + onDone: (text: string) => void; + onError: (message: string) => void; +}; + +/** + * Voice continuation: combine an original dictation with a follow-up + * (appended content or a spoken edit instruction) into one edited text. + */ +export function refineDictation(options: RefineDictationOptions): AnthropicStreamHandle { + return streamAnthropicMessage({ + apiKey: options.apiKey, + model: DEFAULT_LLM_MODEL, + system: REFINE_SYSTEM_PROMPT, + effort: "low", + messages: [ + { + role: "user", + content: `Original dictation:\n${options.original}\n\nFollow-up dictation:\n${options.followup}`, + }, + ], + onTextDelta: options.onTextDelta, + onDone: (result) => { + if (result.stopReason === "refusal") { + options.onError("Anthropic: the model declined this request"); + return; + } + const text = result.text.trim(); + if (!text) { + options.onError("Anthropic: empty response"); + return; + } + options.onDone(text); + }, + onError: options.onError, + }); +} diff --git a/app/ui/apps/voice-input.ts b/app/ui/apps/voice-input.ts index 99cc9e47..cd345577 100644 --- a/app/ui/apps/voice-input.ts +++ b/app/ui/apps/voice-input.ts @@ -1,7 +1,9 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; -import { GESTURE_DOUBLE_CLICK } from "../gestures"; +import { refineDictation, type AnthropicStreamHandle } from "../../native/anthropic"; +import { anthropicApiKeySetting } from "../dashboard-settings"; +import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints } from "../gestures"; import { drawSelectionHighlight } from "../menu"; import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; @@ -11,21 +13,46 @@ const DIALOG_W = G2_LENS_WIDTH - 80; const DIALOG_H = G2_LENS_HEIGHT - 80; const TEXT_MAX_WIDTH = DIALOG_W - 32; const MENU_ROW_H = 20; +const MENU_ROWS = 3; +// After the mic stops, the provider's final transcript can trail in (cloud +// commit round-trip); wait this long for it before refining with what we have. +const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200; + +/** + * The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or + * click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing" + * records a follow-up utterance (always click-to-finish); "refining" streams + * the LLM-merged text, then returns to "menu". + */ +type VoicePhase = "capturing" | "menu" | "continuing" | "refining"; /** * Push-to-talk voice dialog, drawn on top of whatever is already on screen. * The mic runs while the button is held (long-press); releasing it stops the - * mic and shows a Send / Discard menu. Send delivers the transcript to the - * foreground window (e.g. the terminal); Discard (or double-click) closes. + * mic and shows a Send / Continue / Discard menu. Send delivers the transcript + * to the foreground window (e.g. the terminal); Discard (or double-click) + * closes. Continue records another utterance — extra content or a spoken edit + * ("change X to Y", "insert ... after ...") — and merges it into the message + * with an LLM, landing back on the same menu. + * + * When opened from a menu instead of a held button (finishOnClick), there is + * no release event to end the capture, so a single click stops the mic. */ export class VoiceInputLayer implements Layer { + private phase: VoicePhase = "capturing"; private status = "Listening..."; + // The active utterance. displayText() is what the dialog shows and what + // Send delivers; the refine flow also writes the merged result here. private finalizedText = ""; private liveText = ""; + // Snapshot of the message when Continue starts, so a failed or cancelled + // continuation can fall back to it. + private baseText = ""; + /** Whether the mic is running (capture bookkeeping, not UI state). */ private capturing = false; - // Set once the button is released: the mic has stopped and the Send/Discard - // menu is active. - private finished = false; + /** Continuation stopped; waiting for the trailing final transcript. */ + private followupFinalizeTimer: ReturnType | null = null; + private refineHandle: AnthropicStreamHandle | null = null; private menuIndex = 0; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; @@ -34,12 +61,15 @@ export class VoiceInputLayer implements Layer { private readonly actions: LayerActions, private readonly onClosed: () => void, private readonly onSend: (text: string) => void, + private readonly finishOnClick = false, ) {} startCapture(): void { if (this.capturing) return; this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + // The refine stage owns the status line ("Refining...", error text). + if (this.phase === "refining") return; this.status = state.status; this.actions.requestRender(); }); @@ -48,14 +78,18 @@ export class VoiceInputLayer implements Layer { this.actions.requestRender(); } - /** Button released: stop the mic, finalize, and show the Send/Discard menu. */ + /** Button released (or click in finishOnClick mode): stop the mic. */ endCapture(): void { if (!this.capturing) return; + if (this.phase === "continuing") { + this.endContinuationCapture(); + return; + } this.capturing = false; - this.finished = true; + this.phase = "menu"; void this.actions.stopVoiceCapture(); if (this.status.startsWith("Listening")) { - this.status = "Send or discard?"; + this.status = "Send, continue, or discard?"; } this.actions.requestRender(); } @@ -73,58 +107,94 @@ export class VoiceInputLayer implements Layer { image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220); image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130); + const inMenu = this.phase === "menu"; const hasText = this.displayText().trim().length > 0; - // Leave room for the two-row menu when finished. - const textBottom = this.finished ? DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8; + // Leave room for the three-row menu when it is showing. + const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8; const textTop = DIALOG_Y + 56; const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0); - const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)"); + const text = this.displayText() || this.placeholderText(); const wrapped = wrapText(font, text, TEXT_MAX_WIDTH); const firstLine = Math.max(0, wrapped.length - maxLines); for (let index = firstLine; index < wrapped.length; index++) { image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235); } - if (this.finished) { - const labels = ["Send", "Discard"]; - const menuTop = DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 2; - for (let i = 0; i < labels.length; i++) { + if (inMenu) { + const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0; + const rows: Array<{ label: string; dim: boolean }> = [ + // Dim "Send" when there is nothing to send. + { label: "Send", dim: !hasText }, + { label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey }, + { label: "Discard", dim: false }, + ]; + const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2; + for (let i = 0; i < rows.length; i++) { const rowY = menuTop + i * MENU_ROW_H; const selected = i === this.menuIndex; if (selected) { drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6); } - // Dim "Send" when there is nothing to send. - const dim = i === 0 && !hasText; - image.drawText(font, left + 4, rowY + 2, labels[i]!, dim ? 90 : selected ? 255 : 200); + const row = rows[i]!; + image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200); } } else { - image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} close`, 110); + image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, this.hintText(), 110); } return image; } handleInput(event: DashboardInputEvent, ctx: LayerContext): void { - if (!this.finished) { - if (event.type === "double-click") { - ctx.stack.pop(); - } - return; + switch (this.phase) { + case "capturing": + if (event.type === "double-click") { + ctx.stack.pop(); + } else if (this.finishOnClick && event.type === "click") { + this.endCapture(); + } + return; + case "continuing": + if (event.type === "click") { + this.endContinuationCapture(); + } else if (event.type === "double-click") { + this.cancelContinuation("Continuation cancelled"); + } + return; + case "refining": + if (event.type === "double-click") { + this.cancelContinuation("Refinement cancelled"); + } + return; + case "menu": + this.handleMenuInput(event, ctx); + return; } + } + + private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void { switch (event.type) { case "scroll-up": + this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS; + this.actions.requestRender(); + return; case "scroll-down": - // Two options; either scroll direction toggles the selection. - this.menuIndex = (this.menuIndex + 1) % 2; + this.menuIndex = (this.menuIndex + 1) % MENU_ROWS; this.actions.requestRender(); return; case "click": if (this.menuIndex === 0) { const text = this.displayText().trim(); if (text) this.onSend(text); + ctx.stack.pop(); + } else if (this.menuIndex === 1) { + if (anthropicApiKeySetting.get().trim()) { + this.startContinuation(); + } + // No key: the row is disabled; the click does nothing. + } else { + ctx.stack.pop(); } - ctx.stack.pop(); return; case "double-click": ctx.stack.pop(); @@ -134,11 +204,100 @@ export class VoiceInputLayer implements Layer { } } + /** Continue selected: keep the message aside and record a follow-up. */ + private startContinuation(): void { + this.baseText = this.displayText().trim(); + this.finalizedText = ""; + this.liveText = ""; + this.phase = "continuing"; + this.status = "Listening..."; + this.capturing = true; + void this.actions.startVoiceCapture(); + this.actions.requestRender(); + } + + /** Follow-up tap: stop the mic, then refine once the transcript finalizes. */ + private endContinuationCapture(): void { + if (!this.capturing) return; + this.capturing = false; + void this.actions.stopVoiceCapture(); + this.status = "Refining..."; + this.actions.requestRender(); + // The provider's committed transcript arrives shortly after stop; refine + // when it does, or after a timeout with whatever partials we have. + this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); + } + + private beginRefine(): void { + if (this.followupFinalizeTimer !== null) { + clearTimeout(this.followupFinalizeTimer); + this.followupFinalizeTimer = null; + } + if (this.phase !== "continuing") return; + const followup = this.displayText().trim(); + if (!followup) { + this.backToMenu(this.baseText, "No follow-up heard"); + return; + } + this.phase = "refining"; + this.finalizedText = ""; + this.liveText = ""; + this.status = "Refining..."; + this.actions.requestRender(); + this.refineHandle = refineDictation({ + apiKey: anthropicApiKeySetting.get(), + original: this.baseText, + followup, + onTextDelta: (_delta, textSoFar) => { + this.finalizedText = textSoFar; + this.actions.requestRender(); + }, + onDone: (text) => { + this.refineHandle = null; + this.backToMenu(text, "Send, continue, or discard?"); + }, + onError: (message) => { + this.refineHandle = null; + this.backToMenu(this.baseText, message); + }, + }); + } + + /** Abort a continuation (mic or LLM stage) and restore the prior message. */ + private cancelContinuation(status: string): void { + if (this.capturing) { + this.capturing = false; + void this.actions.stopVoiceCapture(); + } + this.refineHandle?.cancel(); + this.refineHandle = null; + this.backToMenu(this.baseText, status); + } + + private backToMenu(text: string, status: string): void { + if (this.followupFinalizeTimer !== null) { + clearTimeout(this.followupFinalizeTimer); + this.followupFinalizeTimer = null; + } + this.finalizedText = text; + this.liveText = ""; + this.phase = "menu"; + this.menuIndex = 0; + this.status = status; + this.actions.requestRender(); + } + onRemoved(): void { this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; + if (this.followupFinalizeTimer !== null) { + clearTimeout(this.followupFinalizeTimer); + this.followupFinalizeTimer = null; + } + this.refineHandle?.cancel(); + this.refineHandle = null; if (this.capturing) { this.capturing = false; void this.actions.stopVoiceCapture(); @@ -152,13 +311,48 @@ export class VoiceInputLayer implements Layer { return `${this.finalizedText} ${this.liveText}`; } + private placeholderText(): string { + switch (this.phase) { + case "capturing": + return "Listening..."; + case "continuing": + return "Say more, or describe an edit..."; + case "refining": + return "Refining..."; + case "menu": + return "(no speech detected)"; + } + } + + private hintText(): string { + switch (this.phase) { + case "capturing": + return this.finishOnClick + ? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]]) + : `${GESTURE_DOUBLE_CLICK} close`; + case "continuing": + return gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "cancel"]]); + case "refining": + default: + return `${GESTURE_DOUBLE_CLICK} cancel`; + } + } + private onTranscript(event: VoiceTranscriptEvent): void { + // The refine stream owns the text buffers once it starts; a transcript + // that trails in after that point is stale. + if (this.phase === "refining") return; if (event.isFinal) { const finalText = event.text.trim() || this.liveText.trim(); if (finalText) { this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; } this.liveText = ""; + if (this.phase === "continuing" && this.followupFinalizeTimer !== null) { + // The follow-up finalized; no need to keep waiting. + this.beginRefine(); + return; + } } else { this.liveText = event.text.trim(); } diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 8a60c0cf..01378b94 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -261,6 +261,16 @@ export const elevenLabsApiKeySetting = new ConfigSettingString({ formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), }); +export const anthropicApiKeySetting = new ConfigSettingString({ + id: "anthropic-api-key", + label: "Anthropic key", + storageKey: "llm.anthropicApiKey", + defaultValue: "", + editorTitle: "Anthropic API key", + glassesEditTitle: "Edit Anthropic key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), +}); + export const terminalHostSetting = new ConfigSettingString({ id: "terminal-host", label: "Host", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 2bd2fa8d..3a87d19f 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -2,6 +2,7 @@ import { getDefaultSmallFont } from "../../graphics/bdffont"; import { getDashboardLogo } from "../../graphics/logo"; import { shell } from "../shell/shell"; import { + anthropicApiKeySetting, batteryDisplayModeSetting, elevenLabsApiKeySetting, enumSettingMenuItem, @@ -47,7 +48,14 @@ function settingsSections(): SettingsSection[] { }, { label: "Voice", - items: [enumSettingMenuItem(voiceProviderSetting), textSettingMenuItem(elevenLabsApiKeySetting)], + items: [enumSettingMenuItem(voiceProviderSetting)], + }, + { + label: "API Keys", + items: [ + textSettingMenuItem(elevenLabsApiKeySetting), + textSettingMenuItem(anthropicApiKeySetting), + ], }, { label: "Integrations", diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 10b68685..20db9916 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -422,7 +422,7 @@ class Shell { next?.requestRender(); } - private openVoiceDialog(): void { + private openVoiceDialog(finishOnClick = false): void { const layer = new VoiceInputLayer( this.config.actions, () => { @@ -431,6 +431,7 @@ class Shell { } }, (text) => this.sendTextToForegroundWindow(text), + finishOnClick, ); this.activeVoiceLayer = layer; this.stack.push(layer); @@ -447,8 +448,14 @@ class Shell { const items: MenuItem[] = [ { label: "Voice input", - onSelect: () => { - this.openVoiceDialog(); + onSelect: (ctx) => { + // Pop the menu first (its onRemoved yields focus to the sidebar), + // then take focus back: the transcript is aimed at the window the + // menu was opened over. The button is no longer held here, so the + // dialog finishes on click instead of long-press-release. + ctx.stack.pop(); + this.focus = "window"; + this.openVoiceDialog(true); }, }, ]; diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index 895385bf..b8e99603 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -20,6 +20,11 @@ export type WorkerAppMessage = export type WorkerAppReply = | { type: "yield-focus"; windowId: string } + | { + /** Foreground and focus one of the app's existing windows. */ + type: "focus-window"; + windowId: string; + } | { /** App-initiated window (e.g. a terminal view opened from the hub list). */ type: "open-window-request"; @@ -84,6 +89,12 @@ export class WorkerAppHost { shell.yieldFocusToSidebar(); } break; + case "focus-window": + if (this.openWindows.has(message.windowId)) { + shell.focusWindow(message.windowId); + this.options.requestShellRender(); + } + break; case "open-window-request": this.openWindow({ windowId: message.windowId, diff --git a/app/workers/terminal-app.worker.ts b/app/workers/terminal-app.worker.ts index e0d4966a..6aeee0b2 100644 --- a/app/workers/terminal-app.worker.ts +++ b/app/workers/terminal-app.worker.ts @@ -173,6 +173,7 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb if (pendingView) { pendingViews.delete(windowId); windows.set(windowId, createViewWindow(windowId, surfaceId, pendingView.socket, pendingView.label)); + renderHubWindows(); return; } windows.set(windowId, { @@ -199,6 +200,9 @@ function closeWindow(windowId: string): void { window.client.stop(); } windows.delete(windowId); + if (window.kind === "view") { + renderHubWindows(); + } } function clientOptions() { @@ -401,9 +405,17 @@ function hubItems(): HubItem[] { if (phase === "connected" || phase === "attached") { for (const session of state?.sessions ?? []) { + const openWindowId = viewWindowIdForSocket(session.socket); items.push({ - label: sessionLabel(session), - onSelect: () => openViewWindowFor(session), + label: openWindowId ? `${sessionLabel(session)} [open]` : sessionLabel(session), + onSelect: () => { + const windowId = viewWindowIdForSocket(session.socket); + if (windowId) { + post({ type: "focus-window", windowId }); + } else { + openViewWindowFor(session); + } + }, }); } if (!state?.sessions.length) { @@ -445,6 +457,21 @@ function handleHubInput(window: HubWindow, event: DashboardInputEvent, frameId: } } +/** + * Window id of the view already showing this session's terminal, if any. + * Includes views that were requested but whose surface hasn't opened yet, so + * a quick double-select can't spawn two windows for one session. + */ +function viewWindowIdForSocket(socket: string): string | null { + for (const window of windows.values()) { + if (window.kind === "view" && window.socket === socket) return window.windowId; + } + for (const [windowId, pending] of pendingViews) { + if (pending.socket === socket) return windowId; + } + return null; +} + function openViewWindowFor(session: G2MirrorSession): void { const windowId = `terminal:view:${nextViewSerial++}`; const label = sessionLabel(session); @@ -595,8 +622,8 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { function sessionLabel(session: G2MirrorSession): string { if (session.title) { - return `${session.title} (pid ${session.pid})`; + return session.title; } const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/"); - return `${hint || "session"} (pid ${session.pid})`; + return hint || "session"; }