forked from Blendman974/faceclaw
Translate app: sound level meter; translate unfinished segments
This commit is contained in:
1 parent
bf2addd287
commit
0b7d93a3f1
25 files changed
+679
-175
No files matched your search
@@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment";
|
||||
import { generateSessionInsights } from "./speaker-insights";
|
||||
import { speakerRegistry, type SpeakerMatch } from "./speakers";
|
||||
import { postAmbientCard } from "../../ui/shell/ambient-cards";
|
||||
import { pcmLevel, rmsToLevel } from "../../util/audio-level";
|
||||
import { formatRelativeTime } from "../../util/date-util";
|
||||
import { deviceLanguage, identifyLanguage, translateText } from "./translate";
|
||||
|
||||
@@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig {
|
||||
return { ...config, channelMask: 0b11 };
|
||||
}
|
||||
|
||||
function pcmLevel(pcm: Uint8Array): number {
|
||||
let sumSquares = 0;
|
||||
const count = pcm.length / 2;
|
||||
if (count === 0) return 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
if (value >= 0x8000) value -= 0x10000;
|
||||
sumSquares += value * value;
|
||||
}
|
||||
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
|
||||
}
|
||||
|
||||
function floatLevel(samples: Float32Array): number {
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
|
||||
return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length)));
|
||||
}
|
||||
|
||||
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
|
||||
function rmsToLevel(rms: number): number {
|
||||
if (rms <= 0) return 0;
|
||||
const db = 20 * Math.log10(rms);
|
||||
return Math.max(0, Math.min(1, (db + 60) / 60));
|
||||
}
|
||||
|
||||
function floatToPcm16(samples: Float32Array): Uint8Array {
|
||||
const out = new Uint8Array(samples.length * 2);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
|
||||
@@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer {
|
||||
private detail = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private readonly meter = new MicLevelMeter();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
@@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer {
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
@@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer {
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
@@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { TranslateLayer } from "./translate";
|
||||
import { resolveTranslationModel } from "./translate-llm";
|
||||
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
|
||||
import {
|
||||
resolveTranslateLanguage,
|
||||
resolveTranslateProvider,
|
||||
translateLanguageSetting,
|
||||
translateProviderSetting,
|
||||
translateShowOriginalSetting,
|
||||
} from "./translate-settings";
|
||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
@@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & {
|
||||
* provider (Translate settings), which a shared capture switches to.
|
||||
*/
|
||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||
// Checks the setup and (re)starts capture with the chosen provider: at
|
||||
// launch, when the provider setting changes, and on a tap after a problem.
|
||||
// Checks the setup and (re)starts capture with the chosen provider and
|
||||
// language: at launch, when either setting changes, and on a tap after a
|
||||
// problem.
|
||||
const startCapture = () => {
|
||||
const provider = resolveTranslateProvider();
|
||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||
@@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
return;
|
||||
}
|
||||
options.startContinuousVoiceCapture({ multilingualProvider: provider });
|
||||
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
};
|
||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
||||
@@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Translate settings", [
|
||||
enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }),
|
||||
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
|
||||
toggleSettingMenuItem(translateShowOriginalSetting),
|
||||
]);
|
||||
@@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
||||
startCapture();
|
||||
return app;
|
||||
}
|
||||
@@ -29,6 +29,8 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null {
|
||||
export type TranslationRequest = {
|
||||
speech: string;
|
||||
context: TranslationTurn[];
|
||||
/** English name of the spoken language when the user set one, else "". */
|
||||
language: string;
|
||||
onProgress: (textSoFar: string) => void;
|
||||
onDone: (translation: string) => void;
|
||||
onError: (message: string) => void;
|
||||
@@ -41,7 +43,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
system: TRANSLATE_SYSTEM_PROMPT,
|
||||
effort: model.effort,
|
||||
maxTokens: 1024,
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }],
|
||||
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
|
||||
onDone: (result) => {
|
||||
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
|
||||
|
||||
@@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>
|
||||
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Spoken languages Translate can be told to expect, by ISO 639-1 code (the
|
||||
* form every multilingual provider accepts), in picker order.
|
||||
*/
|
||||
const SPOKEN_LANGUAGES = {
|
||||
ar: "Arabic",
|
||||
zh: "Chinese",
|
||||
nl: "Dutch",
|
||||
fr: "French",
|
||||
de: "German",
|
||||
el: "Greek",
|
||||
he: "Hebrew",
|
||||
hi: "Hindi",
|
||||
id: "Indonesian",
|
||||
it: "Italian",
|
||||
ja: "Japanese",
|
||||
ko: "Korean",
|
||||
pl: "Polish",
|
||||
pt: "Portuguese",
|
||||
ru: "Russian",
|
||||
es: "Spanish",
|
||||
sv: "Swedish",
|
||||
th: "Thai",
|
||||
tr: "Turkish",
|
||||
uk: "Ukrainian",
|
||||
vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES;
|
||||
|
||||
export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({
|
||||
id: "translate-language",
|
||||
label: "Spoken language",
|
||||
storageKey: "translate.language",
|
||||
defaultValue: "auto",
|
||||
values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])],
|
||||
formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]),
|
||||
description:
|
||||
"The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.",
|
||||
});
|
||||
|
||||
/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */
|
||||
export function resolveTranslateLanguage(): SpokenLanguage | "" {
|
||||
const value = translateLanguageSetting.get();
|
||||
return value === "auto" ? "" : value;
|
||||
}
|
||||
|
||||
/** The English name of a spoken language, for the translation prompt. */
|
||||
export function spokenLanguageName(language: SpokenLanguage): string {
|
||||
return SPOKEN_LANGUAGES[language];
|
||||
}
|
||||
|
||||
export const translateShowOriginalSetting = new ConfigSettingBoolean({
|
||||
id: "translate-show-original",
|
||||
label: "Show original",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
||||
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
||||
import { translateShowOriginalSetting } from "./translate-settings";
|
||||
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
||||
|
||||
const NO_MODEL_NOTICE = "No language model is set up.";
|
||||
|
||||
/**
|
||||
* Live translation view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranslateAppWindow); this layer feeds finalized speech to the
|
||||
* assistant's model one stretch at a time, shows the English as it streams
|
||||
* in, and saves the session to Downloads from the app menu.
|
||||
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
||||
* model one block at a time (provisionally while a block is still being
|
||||
* spoken), shows the English as it streams in, and saves the session to
|
||||
* Downloads from the app menu.
|
||||
*/
|
||||
export class TranslateLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -30,7 +32,8 @@ export class TranslateLayer implements Layer {
|
||||
private problem: string | null = null;
|
||||
private readonly model = new TranslationModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private activeJob: TranslationJob | null = null;
|
||||
private readonly meter = new MicLevelMeter();
|
||||
// The request in flight (the model tracks which job it is).
|
||||
private activeHandle: LlmStreamHandle | null = null;
|
||||
private requestRender: () => void = () => {};
|
||||
private removed = false;
|
||||
@@ -40,10 +43,16 @@ export class TranslateLayer implements Layer {
|
||||
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
||||
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.requestRender = requestRender;
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.model.accept(event);
|
||||
if (this.model.accept(event)) {
|
||||
// The request in flight translates text the transcript has replaced.
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
}
|
||||
this.pump();
|
||||
requestRender();
|
||||
});
|
||||
@@ -68,9 +77,9 @@ export class TranslateLayer implements Layer {
|
||||
const measure = (line: string) => font.measureText(line);
|
||||
const maxWidth = width - 64;
|
||||
|
||||
image.drawText(font, 24, 20, "Translate", 200);
|
||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
||||
image.drawText(font, 24, 40, this.saveNotice || status, 110);
|
||||
const meter = this.listening && !this.problem ? this.meter : null;
|
||||
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
|
||||
|
||||
let lines: TranslationLine[];
|
||||
if (this.problem) {
|
||||
@@ -84,7 +93,6 @@ export class TranslateLayer implements Layer {
|
||||
}
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
||||
@@ -119,7 +127,7 @@ export class TranslateLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.activeJob = null;
|
||||
this.meter.stop();
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
@@ -139,7 +147,10 @@ export class TranslateLayer implements Layer {
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
/** Start translating the next waiting stretch of speech, unless one is in flight. */
|
||||
/**
|
||||
* Start the next translation request (a final block, else the live block's
|
||||
* provisional one), unless one is in flight.
|
||||
*/
|
||||
private pump(): void {
|
||||
if (this.removed) return;
|
||||
const job = this.model.nextJob();
|
||||
@@ -147,36 +158,34 @@ export class TranslateLayer implements Layer {
|
||||
const llm = resolveTranslationModel();
|
||||
if (!llm) {
|
||||
// The model went away mid-session (a key was cleared): keep the original.
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = NO_MODEL_NOTICE;
|
||||
this.pump();
|
||||
return;
|
||||
}
|
||||
this.activeJob = job;
|
||||
const language = resolveTranslateLanguage();
|
||||
const handle = streamTranslation(llm, {
|
||||
speech: job.segment.source,
|
||||
speech: job.source,
|
||||
context: job.context,
|
||||
language: language ? spokenLanguageName(language) : "",
|
||||
onProgress: (textSoFar) => {
|
||||
if (this.activeJob !== job) return;
|
||||
this.model.progress(job.segment, textSoFar);
|
||||
this.requestRender();
|
||||
if (this.model.progress(job, textSoFar)) this.requestRender();
|
||||
},
|
||||
onDone: (translation) => this.settle(job, () => {
|
||||
this.model.complete(job.segment, translation);
|
||||
this.model.complete(job, translation);
|
||||
this.translationError = "";
|
||||
}),
|
||||
onError: (message) => this.settle(job, () => {
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = `Translation failed: ${message}`;
|
||||
}),
|
||||
});
|
||||
// A provider that fails synchronously has already settled (and moved on).
|
||||
if (this.activeJob === job) this.activeHandle = handle;
|
||||
if (this.model.isActive(job)) this.activeHandle = handle;
|
||||
}
|
||||
|
||||
private settle(job: TranslationJob, apply: () => void): void {
|
||||
if (this.activeJob !== job) return;
|
||||
this.activeJob = null;
|
||||
if (!this.model.isActive(job)) return;
|
||||
this.activeHandle = null;
|
||||
apply();
|
||||
this.pump();
|
||||
|
||||
@@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
|
||||
type TranscriptEvent = { text: string; isFinal: boolean };
|
||||
|
||||
export type TranslationState = "pending" | "translating" | "done" | "failed";
|
||||
|
||||
/** One stretch of finalized speech and its English translation. */
|
||||
/**
|
||||
* One block of speech (up to a pause) and its English translation. The last
|
||||
* block may still be live, its transcript revising; it is translated
|
||||
* provisionally as it grows, then again once final if its text changed.
|
||||
*/
|
||||
export type TranslationSegment = {
|
||||
source: string;
|
||||
/** The translation so far (streamed while translating). */
|
||||
/** Committed by the transcriber; a live block's source still revises. */
|
||||
final: boolean;
|
||||
/** The latest finished translation, of `translatedSource`. */
|
||||
translation: string;
|
||||
state: TranslationState;
|
||||
/** What `translation` translates: null until one finishes, behind `source` while provisional. */
|
||||
translatedSource: string | null;
|
||||
/** The text streamed so far by the request in flight for this block, else null. */
|
||||
streaming: string | null;
|
||||
/** A source whose translation failed; not retried until the source changes. */
|
||||
failedSource: string | null;
|
||||
};
|
||||
|
||||
export type TranslationTurn = { source: string; translation: string };
|
||||
|
||||
/** A segment handed to the model, with the finished turns before it. */
|
||||
/** A request: a block's source as sent, with the finished turns before it. */
|
||||
export type TranslationJob = {
|
||||
segment: TranslationSegment;
|
||||
/** The source sent (a live block's moves on while the request is in flight). */
|
||||
source: string;
|
||||
/** Earlier translated turns, oldest first. */
|
||||
context: TranslationTurn[];
|
||||
};
|
||||
@@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number };
|
||||
// over between requests; bounded so a long session's requests stay small.
|
||||
const CONTEXT_TURNS = 4;
|
||||
const CONTEXT_CHARS = 800;
|
||||
// A live block is translated again once its text has changed length by this
|
||||
// much since the last request, so a trickle of revisions (punctuation, one
|
||||
// corrected word) doesn't spend a request each. A final is always redone.
|
||||
const PROVISIONAL_MIN_CHANGE = 3;
|
||||
|
||||
const TRANSLATION_GRAY = 230;
|
||||
const PROVISIONAL_GRAY = 170;
|
||||
const ORIGINAL_GRAY = 120;
|
||||
const LIVE_GRAY = 90;
|
||||
|
||||
function newSegment(source: string, final: boolean): TranslationSegment {
|
||||
return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null };
|
||||
}
|
||||
|
||||
/** Whether a block's current source still needs a translation request. */
|
||||
function wantsTranslation(segment: TranslationSegment): boolean {
|
||||
const { source, translatedSource } = segment;
|
||||
if (!source || source === translatedSource || source === segment.failedSource) return false;
|
||||
return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Translate app's conversation: finalized speech segments with their
|
||||
* translations, plus the live partial of the utterance in progress.
|
||||
* The Translate app's conversation: blocks of speech with their translations.
|
||||
* Translation is serial (one request at a time, in order, so each can carry
|
||||
* the turns before it as context); finals that arrive while a request is in
|
||||
* flight merge into one pending segment, so a slow model batches the backlog
|
||||
* instead of falling further behind one request per pause.
|
||||
* the turns before it as context): final blocks first, then the live one.
|
||||
* Finals that arrive while a request is in flight merge into one waiting
|
||||
* block, so a slow model batches the backlog instead of falling further
|
||||
* behind one request per pause.
|
||||
*/
|
||||
export class TranslationModel {
|
||||
readonly segments: TranslationSegment[] = [];
|
||||
/** The utterance in progress (replace semantics), not yet final. */
|
||||
live = "";
|
||||
private active: TranslationJob | null = null;
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
/**
|
||||
* Take a transcript event (replace semantics within the live block).
|
||||
* Returns true when it made the request in flight moot (its block was
|
||||
* finalized as other text, or dropped); that request is abandoned here, and
|
||||
* the caller should cancel it.
|
||||
*/
|
||||
accept(event: TranscriptEvent): boolean {
|
||||
const text = event.text.trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
this.live = "";
|
||||
if (!finalText) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
|
||||
else this.segments.push({ source: finalText, translation: "", state: "pending" });
|
||||
const live = last && !last.final ? last : null;
|
||||
if (!event.isFinal) {
|
||||
if (live) live.source = text;
|
||||
else if (text) this.segments.push(newSegment(text, false));
|
||||
return false;
|
||||
}
|
||||
const finalText = text || live?.source || "";
|
||||
if (!live) {
|
||||
if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true));
|
||||
return false;
|
||||
}
|
||||
const moot = this.active?.segment === live && this.active.source !== finalText;
|
||||
if (moot) this.abandonActive();
|
||||
if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) {
|
||||
this.segments.pop();
|
||||
return moot;
|
||||
}
|
||||
live.source = finalText;
|
||||
live.final = true;
|
||||
return moot;
|
||||
}
|
||||
|
||||
/** Take the next segment to translate, or null while one is in flight or none waits. */
|
||||
/** Take the next request to make, or null while one is in flight or nothing waits. */
|
||||
nextJob(): TranslationJob | null {
|
||||
if (this.segments.some((segment) => segment.state === "translating")) return null;
|
||||
const segment = this.segments.find((candidate) => candidate.state === "pending");
|
||||
if (this.active) return null;
|
||||
const segment = this.segments.find((candidate) => wantsTranslation(candidate));
|
||||
if (!segment) return null;
|
||||
segment.state = "translating";
|
||||
return { segment, context: this.contextBefore(segment) };
|
||||
segment.streaming = "";
|
||||
this.active = { segment, source: segment.source, context: this.contextBefore(segment) };
|
||||
return this.active;
|
||||
}
|
||||
|
||||
progress(segment: TranslationSegment, textSoFar: string): void {
|
||||
if (segment.state === "translating") segment.translation = textSoFar;
|
||||
isActive(job: TranslationJob): boolean {
|
||||
return this.active === job;
|
||||
}
|
||||
|
||||
complete(segment: TranslationSegment, translation: string): void {
|
||||
segment.translation = translation;
|
||||
segment.state = "done";
|
||||
/** Streamed text for the request in flight; false when `job` isn't it any more. */
|
||||
progress(job: TranslationJob, textSoFar: string): boolean {
|
||||
if (this.active !== job) return false;
|
||||
job.segment.streaming = textSoFar;
|
||||
return true;
|
||||
}
|
||||
|
||||
fail(segment: TranslationSegment): void {
|
||||
segment.translation = "";
|
||||
segment.state = "failed";
|
||||
complete(job: TranslationJob, translation: string): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.translation = translation;
|
||||
job.segment.translatedSource = job.source;
|
||||
}
|
||||
|
||||
fail(job: TranslationJob): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.failedSource = job.source;
|
||||
}
|
||||
|
||||
private abandonActive(): void {
|
||||
if (this.active) this.active.segment.streaming = null;
|
||||
this.active = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a final to the block before `before` (default: the end) when that
|
||||
* block is final but still waiting for its first request. Returns whether
|
||||
* it did.
|
||||
*/
|
||||
private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean {
|
||||
const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1;
|
||||
const waiting = this.segments[index];
|
||||
if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false;
|
||||
if (this.active?.segment === waiting) return false;
|
||||
waiting.source = `${waiting.source} ${finalText}`;
|
||||
return true;
|
||||
}
|
||||
|
||||
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
|
||||
@@ -87,7 +162,7 @@ export class TranslationModel {
|
||||
let chars = 0;
|
||||
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
|
||||
const earlier = this.segments[index]!;
|
||||
if (earlier.state !== "done" || !earlier.translation) continue;
|
||||
if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue;
|
||||
chars += earlier.source.length + earlier.translation.length;
|
||||
if (turns.length && chars > CONTEXT_CHARS) break;
|
||||
turns.unshift({ source: earlier.source, translation: earlier.translation });
|
||||
@@ -96,9 +171,11 @@ export class TranslationModel {
|
||||
}
|
||||
|
||||
/**
|
||||
* Display lines: each segment's translation, bright; its original, dim,
|
||||
* above it when `showOriginal` is on and in its place until a translation
|
||||
* arrives (or when it failed); then the live partial, dimmer.
|
||||
* Display lines: each block's translation, bright once settled (final, and
|
||||
* of its final text) and dimmer while provisional; its original, dim, above
|
||||
* it when `showOriginal` is on and in its place until a translation
|
||||
* arrives. A redo streams in over the translation it replaces only once it
|
||||
* is as long, so the text doesn't shrink and regrow.
|
||||
*/
|
||||
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
|
||||
const lines: TranslationLine[] = [];
|
||||
@@ -106,21 +183,25 @@ export class TranslationModel {
|
||||
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
|
||||
};
|
||||
for (const segment of this.segments) {
|
||||
const translation = segment.translation.trim();
|
||||
// A finished, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
|
||||
if (translation) push(translation, TRANSLATION_GRAY);
|
||||
if (!segment.source) continue;
|
||||
const settled = segment.final && segment.translatedSource === segment.source;
|
||||
const finished = segment.translation.trim();
|
||||
const streaming = segment.streaming?.trim() ?? "";
|
||||
const translation = streaming.length >= finished.length ? streaming : finished;
|
||||
// A settled, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY);
|
||||
if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY);
|
||||
}
|
||||
if (this.live) push(this.live, LIVE_GRAY);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Plain-text transcript for saving: each original, then its translation. */
|
||||
/** Plain-text transcript for saving: each final original, then its translation. */
|
||||
savedText(): string {
|
||||
return this.segments
|
||||
.filter((segment) => segment.final)
|
||||
.map((segment) => {
|
||||
const translation = segment.translation.trim();
|
||||
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation || translation === segment.source.trim()) return segment.source;
|
||||
return `${segment.source}\n→ ${translation}`;
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user