Add Translate app

This commit is contained in:
James Babcock committed 2026-10-24 16:16:26 -07:00
1 parent 07859c95ad
commit c84d301c78
19 files changed
+926 -57

No files matched your search

+2
View File
@@ -9,6 +9,7 @@ import filesApp from "./files";
import musicApp from "./music";
import nightscoutApp from "./nightscout";
import transcribeApp from "./transcribe";
import translateApp from "./translate";
import teleprompterApp from "./teleprompter";
import microphonesApp from "./microphones";
import notificationsApp from "./notifications";
@@ -44,6 +45,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
musicApp,
nightscoutApp,
transcribeApp,
translateApp,
teleprompterApp,
microphonesApp,
notificationsApp,
+2 -2
View File
@@ -13,8 +13,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
const TRAY_ICON_ID = "transcribe";
// 16x18 microphone glyph for the top-bar tray.
const MIC_ICON = imageFromAsciiArt(
// 16x18 microphone glyph for the top-bar tray (also Translate's).
export const MIC_ICON = imageFromAsciiArt(
[
" #### ",
" ###### ",
+18
View File
@@ -0,0 +1,18 @@
import { type AppDefinition } from "../app-definition";
import { createTranslateAppWindow, TRANSLATE_SURFACE_ID, TRANSLATE_WINDOW_ID } from "./translate-app";
const translateApp: AppDefinition = {
appId: "translate",
title: "Translate",
icon: "languages",
launch: (ctx) =>
ctx.launchInProcessApp(TRANSLATE_WINDOW_ID, TRANSLATE_SURFACE_ID, (options) =>
createTranslateAppWindow({
...options,
startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture("translate", capture),
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("translate"),
}),
),
};
export default translateApp;
+99
View File
@@ -0,0 +1,99 @@
import { TranslateLayer } from "./translate";
import { resolveTranslationModel } from "./translate-llm";
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
import { MIC_ICON } from "../transcribe/transcribe-app";
import { transcriptTimestamp } from "../transcribe/transcribe";
import { type ContinuousCaptureOptions } from "../../ui/layers";
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
export const TRANSLATE_WINDOW_ID = "translate";
export const TRANSLATE_SURFACE_ID = "window:translate";
const TRAY_ICON_ID = "translate";
const NO_PROVIDER_PROBLEM =
"Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice).";
const NO_MODEL_PROBLEM = global.isIOS
? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)."
: "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant).";
export type TranslateAppOptions = InProcessAppOptions & {
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
startContinuousVoiceCapture: (options: ContinuousCaptureOptions) => void;
stopContinuousVoiceCapture: () => void;
};
/**
* The Translate app: live speech in any language, translated into English by
* the assistant's model, in its own window. Like Transcribe it holds
* continuous mic capture while open, but always with a multilingual cloud
* provider (Translate settings), which a shared capture switches to.
*/
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
// Checks the setup and (re)starts capture with the chosen provider: at
// launch, when the provider setting changes, and on a tap after a problem.
const startCapture = () => {
const provider = resolveTranslateProvider();
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
layer.setProblem(problem);
if (!provider || problem) {
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
return;
}
options.startContinuousVoiceCapture({ multilingualProvider: provider });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
};
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
const app = createInProcessWindow({
appId: "translate",
windowId: TRANSLATE_WINDOW_ID,
title: "Translate",
iconLetter: "Tl",
icon: "languages",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [
{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
},
},
{
label: "Translate settings",
onSelect: (ctx) => {
ctx.stack.pop();
openSettingsSubMenu(ctx, "Translate settings", [
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
toggleSettingMenuItem(translateShowOriginalSetting),
]);
},
},
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
},
});
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender);
startCapture();
return app;
}
+65
View File
@@ -0,0 +1,65 @@
/**
* Streamed translation against the assistant's configured model (Settings >
* Assistant). Each call translates one stretch of speech into English; see
* TranslationModel for how stretches are batched and given context.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts";
import { type TranslationTurn } from "./translation-model";
/**
* The model Translate uses, or null when none is configured. Hermes is an
* agent with its own tools and memory rather than a translator (every
* utterance would land in its session), so it falls back to Auto: the model
* for whichever API key is set, else the on-phone model.
*/
export function resolveTranslationModel(): ResolvedAssistantModel | null {
const selection = assistantModelSetting.get();
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
}
export type TranslationRequest = {
speech: string;
context: TranslationTurn[];
onProgress: (textSoFar: string) => void;
onDone: (translation: string) => void;
onError: (message: string) => void;
};
export function streamTranslation(model: ResolvedAssistantModel, request: TranslationRequest): LlmStreamHandle {
const options: LlmStreamOptions = {
apiKey: model.apiKey,
model: model.model,
system: TRANSLATE_SYSTEM_PROMPT,
effort: model.effort,
maxTokens: 1024,
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
onDone: (result) => {
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
else request.onDone(cleanTranslation(result.text));
},
onError: request.onError,
};
switch (model.provider) {
case "openai":
return streamOpenAiResponse(options);
case "local":
return streamLocalQwen(options);
default:
return streamAnthropicMessage(options);
}
}
/** The context turns are tagged <english>; a model may echo the tag around its reply. */
function cleanTranslation(text: string): string {
return text.replace(/^\s*<english>/i, "").replace(/<\/english>\s*$/i, "").trim();
}
+56
View File
@@ -0,0 +1,56 @@
import { ConfigSettingBoolean, ConfigSettingEnum, voiceProviderSetting, type VoiceProvider } from "../../ui/dashboard-settings";
/**
* The transcription providers that recognize any spoken language, in picker
* order. The on-device engines can't: Moonshine and Whisper base.en are
* English-only, and Apple's recognizer hears only the iPhone's speech language.
*/
const MULTILINGUAL_PROVIDERS = ["elevenlabs", "whisper", "soniox", "whisper-server"] as const satisfies readonly VoiceProvider[];
export type MultilingualProvider = (typeof MULTILINGUAL_PROVIDERS)[number];
export type TranslateProvider = "auto" | MultilingualProvider;
function isMultilingual(provider: VoiceProvider): provider is MultilingualProvider {
return (MULTILINGUAL_PROVIDERS as readonly VoiceProvider[]).includes(provider);
}
/** Whether the provider has its API key (or, for Whisper server, its URL). */
function isSetUp(provider: MultilingualProvider): boolean {
return !voiceProviderSetting.isDisabled(provider);
}
/**
* The provider Translate transcribes with, or null when none is set up. Auto
* uses the main Transcription Provider when it is multilingual, else the
* first multilingual provider that is set up.
*/
export function resolveTranslateProvider(choice: TranslateProvider = translateProviderSetting.get()): MultilingualProvider | null {
if (choice !== "auto") return isSetUp(choice) ? choice : null;
const main = voiceProviderSetting.get();
if (isMultilingual(main) && isSetUp(main)) return main;
return MULTILINGUAL_PROVIDERS.find(isSetUp) ?? null;
}
export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>({
id: "translate-provider",
label: "Transcription provider",
storageKey: "translate.provider",
defaultValue: "auto",
values: ["auto", ...MULTILINGUAL_PROVIDERS],
formatValue: (value) => {
if (value !== "auto") return voiceProviderSetting.displayValue(value);
const resolved = resolveTranslateProvider("auto");
return `Auto (${resolved ? voiceProviderSetting.displayValue(resolved) : "none set up"})`;
},
isDisabled: (value) => value !== "auto" && !isSetUp(value),
description:
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
});
export const translateShowOriginalSetting = new ConfigSettingBoolean({
id: "translate-show-original",
label: "Show original",
storageKey: "translate.showOriginal",
defaultValue: false,
description: "Show what was said, dimmed, above each translation.",
});
+185
View File
@@ -0,0 +1,185 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
import { type InputEvent } from "../../ui/gestures";
import { type LlmStreamHandle } from "../../assistant/llm-protocol";
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
import { translateShowOriginalSetting } from "./translate-settings";
const NO_MODEL_NOTICE = "No language model is set up.";
/**
* Live translation view. Continuous mic capture is owned by the app wrapper
* (createTranslateAppWindow); this layer feeds finalized speech to the
* assistant's model one stretch at a time, shows the English as it streams
* in, and saves the session to Downloads from the app menu.
*/
export class TranslateLayer implements Layer {
private status = "Starting microphone...";
private listening = false;
private detail = "";
private saveNotice = "";
private translationError = "";
// Why the app can't run (no multilingual provider or no model); shown in
// place of the transcript, and a tap asks the app wrapper to check again.
private problem: string | null = null;
private readonly model = new TranslationModel();
private readonly scroll = new TranscriptScroll();
private activeJob: TranslationJob | null = null;
private activeHandle: LlmStreamHandle | null = null;
private requestRender: () => void = () => {};
private removed = false;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
constructor(private readonly stamp: string, private readonly retry: () => void) {}
start(requestRender: () => void): void {
this.requestRender = requestRender;
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.model.accept(event);
this.pump();
requestRender();
});
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
this.listening = state.listening;
this.detail = state.detail;
this.saveNotice = "";
requestRender();
});
}
setProblem(problem: string | null): void {
this.problem = problem;
this.requestRender();
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const measure = (line: string) => font.measureText(line);
const maxWidth = width - 64;
image.drawText(font, 24, 20, "Translate", 200);
const status = this.problem ? "Setup needed" : this.translationError || this.status;
image.drawText(font, 24, 40, this.saveNotice || status, 110);
let lines: TranslationLine[];
if (this.problem) {
lines = wrapTranscribeText(measure, `${this.problem}\n\nTap to check again.`, maxWidth).map((text) => ({ text, gray: 230 }));
} else {
lines = this.model.lines(measure, maxWidth, translateShowOriginalSetting.get());
if (!lines.length) {
const placeholder = this.listening ? "Listening..." : this.detail;
lines = wrapTranscribeText(measure, placeholder, maxWidth).map((text) => ({ text, gray: 230 }));
}
}
const step = lineStep(font) + 2;
const bodyTop = 62;
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
const firstLine = this.scroll.layout(lines.length, bodyLines);
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
const y = bodyTop + (index - firstLine) * step;
image.drawText(font, 32, y, lines[index]!.text, lines[index]!.gray);
}
if (lines.length > bodyLines) {
const trackHeight = height - bodyTop - 6;
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / lines.length));
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (lines.length - bodyLines));
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
}
return image;
}
handleInput(event: InputEvent, ctx: LayerContext): void {
if (event.type === "scroll-up" || event.type === "scroll-down") {
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
ctx.actions.requestRender();
return;
}
if (event.type === "click" && this.problem) {
this.retry();
return;
}
if (event.type === "double-click") {
ctx.stack.pop();
}
}
onRemoved(): void {
this.removed = true;
this.activeJob = null;
this.activeHandle?.cancel();
this.activeHandle = null;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
}
saveTranscript(): void {
const text = this.model.savedText().trim();
if (!text) {
this.saveNotice = "Nothing to save yet.";
return;
}
const filename = `translation-${this.stamp}.txt`;
const path = writeTextToDownloads(filename, `${text}\n`);
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
}
/** Start translating the next waiting stretch of speech, unless one is in flight. */
private pump(): void {
if (this.removed) return;
const job = this.model.nextJob();
if (!job) return;
const llm = resolveTranslationModel();
if (!llm) {
// The model went away mid-session (a key was cleared): keep the original.
this.model.fail(job.segment);
this.translationError = NO_MODEL_NOTICE;
this.pump();
return;
}
this.activeJob = job;
const handle = streamTranslation(llm, {
speech: job.segment.source,
context: job.context,
onProgress: (textSoFar) => {
if (this.activeJob !== job) return;
this.model.progress(job.segment, textSoFar);
this.requestRender();
},
onDone: (translation) => this.settle(job, () => {
this.model.complete(job.segment, translation);
this.translationError = "";
}),
onError: (message) => this.settle(job, () => {
this.model.fail(job.segment);
this.translationError = `Translation failed: ${message}`;
}),
});
// A provider that fails synchronously has already settled (and moved on).
if (this.activeJob === job) this.activeHandle = handle;
}
private settle(job: TranslationJob, apply: () => void): void {
if (this.activeJob !== job) return;
this.activeJob = null;
this.activeHandle = null;
apply();
this.pump();
this.requestRender();
}
}
+129
View File
@@ -0,0 +1,129 @@
import { wrapTranscribeText } from "../transcribe/transcript-model";
type TranscriptEvent = { text: string; isFinal: boolean };
export type TranslationState = "pending" | "translating" | "done" | "failed";
/** One stretch of finalized speech and its English translation. */
export type TranslationSegment = {
source: string;
/** The translation so far (streamed while translating). */
translation: string;
state: TranslationState;
};
export type TranslationTurn = { source: string; translation: string };
/** A segment handed to the model, with the finished turns before it. */
export type TranslationJob = {
segment: TranslationSegment;
/** Earlier translated turns, oldest first. */
context: TranslationTurn[];
};
export type TranslationLine = { text: string; gray: number };
// Earlier turns sent with each request, so pronouns, names, and topic carry
// over between requests; bounded so a long session's requests stay small.
const CONTEXT_TURNS = 4;
const CONTEXT_CHARS = 800;
const TRANSLATION_GRAY = 230;
const ORIGINAL_GRAY = 120;
const LIVE_GRAY = 90;
/**
* The Translate app's conversation: finalized speech segments with their
* translations, plus the live partial of the utterance in progress.
* Translation is serial (one request at a time, in order, so each can carry
* the turns before it as context); finals that arrive while a request is in
* flight merge into one pending segment, so a slow model batches the backlog
* instead of falling further behind one request per pause.
*/
export class TranslationModel {
readonly segments: TranslationSegment[] = [];
/** The utterance in progress (replace semantics), not yet final. */
live = "";
accept(event: TranscriptEvent): void {
const text = event.text.trim();
if (!event.isFinal) {
this.live = text;
return;
}
const finalText = text || this.live;
this.live = "";
if (!finalText) return;
const last = this.segments[this.segments.length - 1];
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
else this.segments.push({ source: finalText, translation: "", state: "pending" });
}
/** Take the next segment to translate, or null while one is in flight or none waits. */
nextJob(): TranslationJob | null {
if (this.segments.some((segment) => segment.state === "translating")) return null;
const segment = this.segments.find((candidate) => candidate.state === "pending");
if (!segment) return null;
segment.state = "translating";
return { segment, context: this.contextBefore(segment) };
}
progress(segment: TranslationSegment, textSoFar: string): void {
if (segment.state === "translating") segment.translation = textSoFar;
}
complete(segment: TranslationSegment, translation: string): void {
segment.translation = translation;
segment.state = "done";
}
fail(segment: TranslationSegment): void {
segment.translation = "";
segment.state = "failed";
}
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
const turns: TranslationTurn[] = [];
let chars = 0;
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
const earlier = this.segments[index]!;
if (earlier.state !== "done" || !earlier.translation) continue;
chars += earlier.source.length + earlier.translation.length;
if (turns.length && chars > CONTEXT_CHARS) break;
turns.unshift({ source: earlier.source, translation: earlier.translation });
}
return turns;
}
/**
* Display lines: each segment's translation, bright; its original, dim,
* above it when `showOriginal` is on and in its place until a translation
* arrives (or when it failed); then the live partial, dimmer.
*/
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
const lines: TranslationLine[] = [];
const push = (text: string, gray: number) => {
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
};
for (const segment of this.segments) {
const translation = segment.translation.trim();
// A finished, empty translation is noise the model chose to drop.
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
if (translation) push(translation, TRANSLATION_GRAY);
}
if (this.live) push(this.live, LIVE_GRAY);
return lines;
}
/** Plain-text transcript for saving: each original, then its translation. */
savedText(): string {
return this.segments
.map((segment) => {
const translation = segment.translation.trim();
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
if (!translation || translation === segment.source.trim()) return segment.source;
return `${segment.source}\n→ ${translation}`;
})
.join("\n\n");
}
}