forked from Blendman974/faceclaw
Add Translate app
This commit is contained in:
1 parent
07859c95ad
commit
c84d301c78
19 files changed
+926
-57
No files matched your search
@@ -9,6 +9,7 @@ import filesApp from "./files";
|
||||
import musicApp from "./music";
|
||||
import nightscoutApp from "./nightscout";
|
||||
import transcribeApp from "./transcribe";
|
||||
import translateApp from "./translate";
|
||||
import teleprompterApp from "./teleprompter";
|
||||
import microphonesApp from "./microphones";
|
||||
import notificationsApp from "./notifications";
|
||||
@@ -44,6 +45,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
|
||||
musicApp,
|
||||
nightscoutApp,
|
||||
transcribeApp,
|
||||
translateApp,
|
||||
teleprompterApp,
|
||||
microphonesApp,
|
||||
notificationsApp,
|
||||
|
||||
@@ -13,8 +13,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
|
||||
|
||||
const TRAY_ICON_ID = "transcribe";
|
||||
|
||||
// 16x18 microphone glyph for the top-bar tray.
|
||||
const MIC_ICON = imageFromAsciiArt(
|
||||
// 16x18 microphone glyph for the top-bar tray (also Translate's).
|
||||
export const MIC_ICON = imageFromAsciiArt(
|
||||
[
|
||||
" #### ",
|
||||
" ###### ",
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTranslateAppWindow, TRANSLATE_SURFACE_ID, TRANSLATE_WINDOW_ID } from "./translate-app";
|
||||
|
||||
const translateApp: AppDefinition = {
|
||||
appId: "translate",
|
||||
title: "Translate",
|
||||
icon: "languages",
|
||||
launch: (ctx) =>
|
||||
ctx.launchInProcessApp(TRANSLATE_WINDOW_ID, TRANSLATE_SURFACE_ID, (options) =>
|
||||
createTranslateAppWindow({
|
||||
...options,
|
||||
startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture("translate", capture),
|
||||
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("translate"),
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
export default translateApp;
|
||||
@@ -0,0 +1,99 @@
|
||||
import { TranslateLayer } from "./translate";
|
||||
import { resolveTranslationModel } from "./translate-llm";
|
||||
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
|
||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
|
||||
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
type InProcessAppOptions,
|
||||
type InProcessWindow,
|
||||
} from "../../ui/shell/in-process-window";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
|
||||
export const TRANSLATE_WINDOW_ID = "translate";
|
||||
export const TRANSLATE_SURFACE_ID = "window:translate";
|
||||
|
||||
const TRAY_ICON_ID = "translate";
|
||||
|
||||
const NO_PROVIDER_PROBLEM =
|
||||
"Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice).";
|
||||
const NO_MODEL_PROBLEM = global.isIOS
|
||||
? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)."
|
||||
: "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant).";
|
||||
|
||||
export type TranslateAppOptions = InProcessAppOptions & {
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
startContinuousVoiceCapture: (options: ContinuousCaptureOptions) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The Translate app: live speech in any language, translated into English by
|
||||
* the assistant's model, in its own window. Like Transcribe it holds
|
||||
* continuous mic capture while open, but always with a multilingual cloud
|
||||
* provider (Translate settings), which a shared capture switches to.
|
||||
*/
|
||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||
// Checks the setup and (re)starts capture with the chosen provider: at
|
||||
// launch, when the provider setting changes, and on a tap after a problem.
|
||||
const startCapture = () => {
|
||||
const provider = resolveTranslateProvider();
|
||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||
layer.setProblem(problem);
|
||||
if (!provider || problem) {
|
||||
options.stopContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
return;
|
||||
}
|
||||
options.startContinuousVoiceCapture({ multilingualProvider: provider });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
};
|
||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
||||
const app = createInProcessWindow({
|
||||
appId: "translate",
|
||||
windowId: TRANSLATE_WINDOW_ID,
|
||||
title: "Translate",
|
||||
iconLetter: "Tl",
|
||||
icon: "languages",
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [
|
||||
{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Translate settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Translate settings", [
|
||||
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
|
||||
toggleSettingMenuItem(translateShowOriginalSetting),
|
||||
]);
|
||||
},
|
||||
},
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
options.stopContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
startCapture();
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Streamed translation against the assistant's configured model (Settings >
|
||||
* Assistant). Each call translates one stretch of speech into English; see
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts";
|
||||
import { type TranslationTurn } from "./translation-model";
|
||||
|
||||
/**
|
||||
* The model Translate uses, or null when none is configured. Hermes is an
|
||||
* agent with its own tools and memory rather than a translator (every
|
||||
* utterance would land in its session), so it falls back to Auto: the model
|
||||
* for whichever API key is set, else the on-phone model.
|
||||
*/
|
||||
export function resolveTranslationModel(): ResolvedAssistantModel | null {
|
||||
const selection = assistantModelSetting.get();
|
||||
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
}
|
||||
|
||||
export type TranslationRequest = {
|
||||
speech: string;
|
||||
context: TranslationTurn[];
|
||||
onProgress: (textSoFar: string) => void;
|
||||
onDone: (translation: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export function streamTranslation(model: ResolvedAssistantModel, request: TranslationRequest): LlmStreamHandle {
|
||||
const options: LlmStreamOptions = {
|
||||
apiKey: model.apiKey,
|
||||
model: model.model,
|
||||
system: TRANSLATE_SYSTEM_PROMPT,
|
||||
effort: model.effort,
|
||||
maxTokens: 1024,
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
|
||||
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
|
||||
onDone: (result) => {
|
||||
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
|
||||
else request.onDone(cleanTranslation(result.text));
|
||||
},
|
||||
onError: request.onError,
|
||||
};
|
||||
switch (model.provider) {
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
}
|
||||
|
||||
/** The context turns are tagged <english>; a model may echo the tag around its reply. */
|
||||
function cleanTranslation(text: string): string {
|
||||
return text.replace(/^\s*<english>/i, "").replace(/<\/english>\s*$/i, "").trim();
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { ConfigSettingBoolean, ConfigSettingEnum, voiceProviderSetting, type VoiceProvider } from "../../ui/dashboard-settings";
|
||||
|
||||
/**
|
||||
* The transcription providers that recognize any spoken language, in picker
|
||||
* order. The on-device engines can't: Moonshine and Whisper base.en are
|
||||
* English-only, and Apple's recognizer hears only the iPhone's speech language.
|
||||
*/
|
||||
const MULTILINGUAL_PROVIDERS = ["elevenlabs", "whisper", "soniox", "whisper-server"] as const satisfies readonly VoiceProvider[];
|
||||
|
||||
export type MultilingualProvider = (typeof MULTILINGUAL_PROVIDERS)[number];
|
||||
export type TranslateProvider = "auto" | MultilingualProvider;
|
||||
|
||||
function isMultilingual(provider: VoiceProvider): provider is MultilingualProvider {
|
||||
return (MULTILINGUAL_PROVIDERS as readonly VoiceProvider[]).includes(provider);
|
||||
}
|
||||
|
||||
/** Whether the provider has its API key (or, for Whisper server, its URL). */
|
||||
function isSetUp(provider: MultilingualProvider): boolean {
|
||||
return !voiceProviderSetting.isDisabled(provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* The provider Translate transcribes with, or null when none is set up. Auto
|
||||
* uses the main Transcription Provider when it is multilingual, else the
|
||||
* first multilingual provider that is set up.
|
||||
*/
|
||||
export function resolveTranslateProvider(choice: TranslateProvider = translateProviderSetting.get()): MultilingualProvider | null {
|
||||
if (choice !== "auto") return isSetUp(choice) ? choice : null;
|
||||
const main = voiceProviderSetting.get();
|
||||
if (isMultilingual(main) && isSetUp(main)) return main;
|
||||
return MULTILINGUAL_PROVIDERS.find(isSetUp) ?? null;
|
||||
}
|
||||
|
||||
export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>({
|
||||
id: "translate-provider",
|
||||
label: "Transcription provider",
|
||||
storageKey: "translate.provider",
|
||||
defaultValue: "auto",
|
||||
values: ["auto", ...MULTILINGUAL_PROVIDERS],
|
||||
formatValue: (value) => {
|
||||
if (value !== "auto") return voiceProviderSetting.displayValue(value);
|
||||
const resolved = resolveTranslateProvider("auto");
|
||||
return `Auto (${resolved ? voiceProviderSetting.displayValue(resolved) : "none set up"})`;
|
||||
},
|
||||
isDisabled: (value) => value !== "auto" && !isSetUp(value),
|
||||
description:
|
||||
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
|
||||
});
|
||||
|
||||
export const translateShowOriginalSetting = new ConfigSettingBoolean({
|
||||
id: "translate-show-original",
|
||||
label: "Show original",
|
||||
storageKey: "translate.showOriginal",
|
||||
defaultValue: false,
|
||||
description: "Show what was said, dimmed, above each translation.",
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
||||
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
||||
import { translateShowOriginalSetting } from "./translate-settings";
|
||||
|
||||
const NO_MODEL_NOTICE = "No language model is set up.";
|
||||
|
||||
/**
|
||||
* Live translation view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranslateAppWindow); this layer feeds finalized speech to the
|
||||
* assistant's model one stretch at a time, shows the English as it streams
|
||||
* in, and saves the session to Downloads from the app menu.
|
||||
*/
|
||||
export class TranslateLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
private listening = false;
|
||||
private detail = "";
|
||||
private saveNotice = "";
|
||||
private translationError = "";
|
||||
// Why the app can't run (no multilingual provider or no model); shown in
|
||||
// place of the transcript, and a tap asks the app wrapper to check again.
|
||||
private problem: string | null = null;
|
||||
private readonly model = new TranslationModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private activeJob: TranslationJob | null = null;
|
||||
private activeHandle: LlmStreamHandle | null = null;
|
||||
private requestRender: () => void = () => {};
|
||||
private removed = false;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
||||
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.requestRender = requestRender;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.model.accept(event);
|
||||
this.pump();
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
});
|
||||
}
|
||||
|
||||
setProblem(problem: string | null): void {
|
||||
this.problem = problem;
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const measure = (line: string) => font.measureText(line);
|
||||
const maxWidth = width - 64;
|
||||
|
||||
image.drawText(font, 24, 20, "Translate", 200);
|
||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
||||
image.drawText(font, 24, 40, this.saveNotice || status, 110);
|
||||
|
||||
let lines: TranslationLine[];
|
||||
if (this.problem) {
|
||||
lines = wrapTranscribeText(measure, `${this.problem}\n\nTap to check again.`, maxWidth).map((text) => ({ text, gray: 230 }));
|
||||
} else {
|
||||
lines = this.model.lines(measure, maxWidth, translateShowOriginalSetting.get());
|
||||
if (!lines.length) {
|
||||
const placeholder = this.listening ? "Listening..." : this.detail;
|
||||
lines = wrapTranscribeText(measure, placeholder, maxWidth).map((text) => ({ text, gray: 230 }));
|
||||
}
|
||||
}
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
||||
const y = bodyTop + (index - firstLine) * step;
|
||||
image.drawText(font, 32, y, lines[index]!.text, lines[index]!.gray);
|
||||
}
|
||||
|
||||
if (lines.length > bodyLines) {
|
||||
const trackHeight = height - bodyTop - 6;
|
||||
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / lines.length));
|
||||
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (lines.length - bodyLines));
|
||||
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
|
||||
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "scroll-up" || event.type === "scroll-down") {
|
||||
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click" && this.problem) {
|
||||
this.retry();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.activeJob = null;
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
}
|
||||
|
||||
saveTranscript(): void {
|
||||
const text = this.model.savedText().trim();
|
||||
if (!text) {
|
||||
this.saveNotice = "Nothing to save yet.";
|
||||
return;
|
||||
}
|
||||
const filename = `translation-${this.stamp}.txt`;
|
||||
const path = writeTextToDownloads(filename, `${text}\n`);
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
/** Start translating the next waiting stretch of speech, unless one is in flight. */
|
||||
private pump(): void {
|
||||
if (this.removed) return;
|
||||
const job = this.model.nextJob();
|
||||
if (!job) return;
|
||||
const llm = resolveTranslationModel();
|
||||
if (!llm) {
|
||||
// The model went away mid-session (a key was cleared): keep the original.
|
||||
this.model.fail(job.segment);
|
||||
this.translationError = NO_MODEL_NOTICE;
|
||||
this.pump();
|
||||
return;
|
||||
}
|
||||
this.activeJob = job;
|
||||
const handle = streamTranslation(llm, {
|
||||
speech: job.segment.source,
|
||||
context: job.context,
|
||||
onProgress: (textSoFar) => {
|
||||
if (this.activeJob !== job) return;
|
||||
this.model.progress(job.segment, textSoFar);
|
||||
this.requestRender();
|
||||
},
|
||||
onDone: (translation) => this.settle(job, () => {
|
||||
this.model.complete(job.segment, translation);
|
||||
this.translationError = "";
|
||||
}),
|
||||
onError: (message) => this.settle(job, () => {
|
||||
this.model.fail(job.segment);
|
||||
this.translationError = `Translation failed: ${message}`;
|
||||
}),
|
||||
});
|
||||
// A provider that fails synchronously has already settled (and moved on).
|
||||
if (this.activeJob === job) this.activeHandle = handle;
|
||||
}
|
||||
|
||||
private settle(job: TranslationJob, apply: () => void): void {
|
||||
if (this.activeJob !== job) return;
|
||||
this.activeJob = null;
|
||||
this.activeHandle = null;
|
||||
apply();
|
||||
this.pump();
|
||||
this.requestRender();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
|
||||
type TranscriptEvent = { text: string; isFinal: boolean };
|
||||
|
||||
export type TranslationState = "pending" | "translating" | "done" | "failed";
|
||||
|
||||
/** One stretch of finalized speech and its English translation. */
|
||||
export type TranslationSegment = {
|
||||
source: string;
|
||||
/** The translation so far (streamed while translating). */
|
||||
translation: string;
|
||||
state: TranslationState;
|
||||
};
|
||||
|
||||
export type TranslationTurn = { source: string; translation: string };
|
||||
|
||||
/** A segment handed to the model, with the finished turns before it. */
|
||||
export type TranslationJob = {
|
||||
segment: TranslationSegment;
|
||||
/** Earlier translated turns, oldest first. */
|
||||
context: TranslationTurn[];
|
||||
};
|
||||
|
||||
export type TranslationLine = { text: string; gray: number };
|
||||
|
||||
// Earlier turns sent with each request, so pronouns, names, and topic carry
|
||||
// over between requests; bounded so a long session's requests stay small.
|
||||
const CONTEXT_TURNS = 4;
|
||||
const CONTEXT_CHARS = 800;
|
||||
|
||||
const TRANSLATION_GRAY = 230;
|
||||
const ORIGINAL_GRAY = 120;
|
||||
const LIVE_GRAY = 90;
|
||||
|
||||
/**
|
||||
* The Translate app's conversation: finalized speech segments with their
|
||||
* translations, plus the live partial of the utterance in progress.
|
||||
* Translation is serial (one request at a time, in order, so each can carry
|
||||
* the turns before it as context); finals that arrive while a request is in
|
||||
* flight merge into one pending segment, so a slow model batches the backlog
|
||||
* instead of falling further behind one request per pause.
|
||||
*/
|
||||
export class TranslationModel {
|
||||
readonly segments: TranslationSegment[] = [];
|
||||
/** The utterance in progress (replace semantics), not yet final. */
|
||||
live = "";
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
const text = event.text.trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
this.live = "";
|
||||
if (!finalText) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
|
||||
else this.segments.push({ source: finalText, translation: "", state: "pending" });
|
||||
}
|
||||
|
||||
/** Take the next segment to translate, or null while one is in flight or none waits. */
|
||||
nextJob(): TranslationJob | null {
|
||||
if (this.segments.some((segment) => segment.state === "translating")) return null;
|
||||
const segment = this.segments.find((candidate) => candidate.state === "pending");
|
||||
if (!segment) return null;
|
||||
segment.state = "translating";
|
||||
return { segment, context: this.contextBefore(segment) };
|
||||
}
|
||||
|
||||
progress(segment: TranslationSegment, textSoFar: string): void {
|
||||
if (segment.state === "translating") segment.translation = textSoFar;
|
||||
}
|
||||
|
||||
complete(segment: TranslationSegment, translation: string): void {
|
||||
segment.translation = translation;
|
||||
segment.state = "done";
|
||||
}
|
||||
|
||||
fail(segment: TranslationSegment): void {
|
||||
segment.translation = "";
|
||||
segment.state = "failed";
|
||||
}
|
||||
|
||||
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
|
||||
const turns: TranslationTurn[] = [];
|
||||
let chars = 0;
|
||||
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
|
||||
const earlier = this.segments[index]!;
|
||||
if (earlier.state !== "done" || !earlier.translation) continue;
|
||||
chars += earlier.source.length + earlier.translation.length;
|
||||
if (turns.length && chars > CONTEXT_CHARS) break;
|
||||
turns.unshift({ source: earlier.source, translation: earlier.translation });
|
||||
}
|
||||
return turns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Display lines: each segment's translation, bright; its original, dim,
|
||||
* above it when `showOriginal` is on and in its place until a translation
|
||||
* arrives (or when it failed); then the live partial, dimmer.
|
||||
*/
|
||||
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
|
||||
const lines: TranslationLine[] = [];
|
||||
const push = (text: string, gray: number) => {
|
||||
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
|
||||
};
|
||||
for (const segment of this.segments) {
|
||||
const translation = segment.translation.trim();
|
||||
// A finished, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
|
||||
if (translation) push(translation, TRANSLATION_GRAY);
|
||||
}
|
||||
if (this.live) push(this.live, LIVE_GRAY);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Plain-text transcript for saving: each original, then its translation. */
|
||||
savedText(): string {
|
||||
return this.segments
|
||||
.map((segment) => {
|
||||
const translation = segment.translation.trim();
|
||||
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation || translation === segment.source.trim()) return segment.source;
|
||||
return `${segment.source}\n→ ${translation}`;
|
||||
})
|
||||
.join("\n\n");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user