Voice: self-hosted Whisper server provider; Transcribe on iOS

- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible
  Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests,
  with URL, model, language and key settings (Voice). Pause-ended spans are
  one request each, finalized in audio order, with provisional text while a
  span is open. See notes/whisper-server.md.
- Transcribe (and other continuous capture) works on iOS with a cloud or
  Whisper-server provider; push-to-talk shares a running capture.
- Transcribe saves each session's audio as transcribe-<time>.wav in Downloads
  (setting "Save Transcribe audio", on by default), written as it is captured
  so a crash leaves a playable file; a saved transcript uses the same stamp.
- Calculator stops continuous listening when its window closes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
FilipAlexanderandClaude Opus 5.5 committed 2026-10-17 16:43:18 +02:00
1 parent ecb6c7f465
commit e2caf28392
24 files changed
+1065 -43

No files matched your search

+3
View File
@@ -116,6 +116,9 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce
},
removeSurface: options.removeSurface,
onClosed: () => {
// Closing keeps the base layer (clearToBase), so stop its continuous
// listening explicitly; otherwise the mic capture outlives the window.
layer.onRemoved();
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
options.onClosed();
},
-1
View File
@@ -1,7 +1,6 @@
/** Native services still outside the current iOS app-screen milestone. */
const unavailable: Record<string, string> = {
music: 'Media-player integration is not available on iOS yet.',
transcribe: 'Voice capture and transcription are not available on iOS yet.',
microphones: 'Glasses microphone capture is not available on iOS yet.',
}
export function iosAppUnavailableReason(appId: string): string | null {
+1 -1
View File
@@ -9,7 +9,7 @@ const transcribeApp: AppDefinition = {
ctx.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) =>
createTranscribeAppWindow({
...options,
startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(),
startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture(capture),
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(),
}),
),
+6 -4
View File
@@ -1,5 +1,5 @@
import { imageFromAsciiArt } from "../../graphics/image";
import { TranscribeLayer } from "./transcribe";
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
import {
createInProcessWindow,
YieldAtRootLayer,
@@ -37,7 +37,7 @@ const MIC_ICON = imageFromAsciiArt(
export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */
startContinuousVoiceCapture: () => void;
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
stopContinuousVoiceCapture: () => void;
};
@@ -47,7 +47,9 @@ export type TranscribeAppOptions = InProcessAppOptions & {
* shows a microphone tray icon in the top bar.
*/
export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow {
const layer = new TranscribeLayer();
// One timestamp per session names both the audio recording and a saved transcript.
const stamp = transcriptTimestamp();
const layer = new TranscribeLayer(stamp);
const app = createInProcessWindow({
appId: "transcribe",
windowId: TRANSCRIBE_WINDOW_ID,
@@ -77,7 +79,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender);
options.startContinuousVoiceCapture();
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
return app;
}
+5 -2
View File
@@ -23,6 +23,9 @@ export class TranscribeLayer implements Layer {
private unsubscribePause: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
/** `stamp` names the saved transcript, matching the session's audio recording. */
constructor(private readonly stamp: string = transcriptTimestamp()) {}
start(requestRender: () => void): void {
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.transcript.accept(event);
@@ -93,13 +96,13 @@ export class TranscribeLayer implements Layer {
this.saveNotice = "Nothing to save yet.";
return;
}
const filename = `transcript-${transcriptTimestamp()}.txt`;
const filename = `transcript-${this.stamp}.txt`;
const path = writeTextToDownloads(filename, `${text}\n`);
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
}
}
function transcriptTimestamp(): string {
export function transcriptTimestamp(): string {
const now = new Date();
const pad = (value: number) => String(value).padStart(2, "0");
return (