Translate app: sound level meter; translate unfinished segments

This commit is contained in:
James Babcock committed 2026-10-25 13:12:25 -07:00
1 parent bf2addd287
commit 0b7d93a3f1
25 files changed
+679 -175

No files matched your search

+1 -19
View File
@@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment";
import { generateSessionInsights } from "./speaker-insights";
import { speakerRegistry, type SpeakerMatch } from "./speakers";
import { postAmbientCard } from "../../ui/shell/ambient-cards";
import { pcmLevel, rmsToLevel } from "../../util/audio-level";
import { formatRelativeTime } from "../../util/date-util";
import { deviceLanguage, identifyLanguage, translateText } from "./translate";
@@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig {
return { ...config, channelMask: 0b11 };
}
function pcmLevel(pcm: Uint8Array): number {
let sumSquares = 0;
const count = pcm.length / 2;
if (count === 0) return 0;
for (let i = 0; i < count; i++) {
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
if (value >= 0x8000) value -= 0x10000;
sumSquares += value * value;
}
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
}
function floatLevel(samples: Float32Array): number {
let sumSquares = 0;
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length)));
}
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
function rmsToLevel(rms: number): number {
if (rms <= 0) return 0;
const db = 20 * Math.log10(rms);
return Math.max(0, Math.min(1, (db + 60) / 60));
}
function floatToPcm16(samples: Float32Array): Uint8Array {
const out = new Uint8Array(samples.length * 2);
for (let i = 0; i < samples.length; i++) {
+1 -1
View File
@@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
});
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender);
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
return app;
+7 -5
View File
@@ -1,6 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
@@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer {
private detail = "";
private readonly transcript = new TranscriptModel();
private readonly scroll = new TranscriptScroll();
private readonly meter = new MicLevelMeter();
private saveNotice = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribePause: (() => void) | null = null;
@@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer {
/** `stamp` names the saved transcript, matching the session's audio recording. */
constructor(private readonly stamp: string = transcriptTimestamp()) {}
start(requestRender: () => void): void {
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
start(requestRender: () => void, isVisible: () => boolean): void {
this.meter.start(requestRender, isVisible);
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.transcript.accept(event);
requestRender();
@@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer {
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
image.drawText(font, 24, 20, "Transcribe", 200);
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
const step = lineStep(font) + 2;
const bodyTop = 62;
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
@@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer {
}
onRemoved(): void {
this.meter.stop();
this.unsubscribePause?.();
this.unsubscribePause = null;
this.unsubscribeTranscript?.();
+13 -5
View File
@@ -1,6 +1,12 @@
import { TranslateLayer } from "./translate";
import { resolveTranslationModel } from "./translate-llm";
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
import {
resolveTranslateLanguage,
resolveTranslateProvider,
translateLanguageSetting,
translateProviderSetting,
translateShowOriginalSetting,
} from "./translate-settings";
import { MIC_ICON } from "../transcribe/transcribe-app";
import { transcriptTimestamp } from "../transcribe/transcribe";
import { type ContinuousCaptureOptions } from "../../ui/layers";
@@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & {
* provider (Translate settings), which a shared capture switches to.
*/
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
// Checks the setup and (re)starts capture with the chosen provider: at
// launch, when the provider setting changes, and on a tap after a problem.
// Checks the setup and (re)starts capture with the chosen provider and
// language: at launch, when either setting changes, and on a tap after a
// problem.
const startCapture = () => {
const provider = resolveTranslateProvider();
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
@@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
shell.setTrayIcon(TRAY_ICON_ID, null);
return;
}
options.startContinuousVoiceCapture({ multilingualProvider: provider });
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
};
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
@@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
onSelect: (ctx) => {
ctx.stack.pop();
openSettingsSubMenu(ctx, "Translate settings", [
enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }),
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
toggleSettingMenuItem(translateShowOriginalSetting),
]);
@@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
});
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender);
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
startCapture();
return app;
}
+3 -1
View File
@@ -29,6 +29,8 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null {
export type TranslationRequest = {
speech: string;
context: TranslationTurn[];
/** English name of the spoken language when the user set one, else "". */
language: string;
onProgress: (textSoFar: string) => void;
onDone: (translation: string) => void;
onError: (message: string) => void;
@@ -41,7 +43,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
system: TRANSLATE_SYSTEM_PROMPT,
effort: model.effort,
maxTokens: 1024,
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }],
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
onDone: (result) => {
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
+52
View File
@@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
});
/**
* Spoken languages Translate can be told to expect, by ISO 639-1 code (the
* form every multilingual provider accepts), in picker order.
*/
const SPOKEN_LANGUAGES = {
ar: "Arabic",
zh: "Chinese",
nl: "Dutch",
fr: "French",
de: "German",
el: "Greek",
he: "Hebrew",
hi: "Hindi",
id: "Indonesian",
it: "Italian",
ja: "Japanese",
ko: "Korean",
pl: "Polish",
pt: "Portuguese",
ru: "Russian",
es: "Spanish",
sv: "Swedish",
th: "Thai",
tr: "Turkish",
uk: "Ukrainian",
vi: "Vietnamese",
} as const;
export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES;
export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({
id: "translate-language",
label: "Spoken language",
storageKey: "translate.language",
defaultValue: "auto",
values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])],
formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]),
description:
"The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.",
});
/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */
export function resolveTranslateLanguage(): SpokenLanguage | "" {
const value = translateLanguageSetting.get();
return value === "auto" ? "" : value;
}
/** The English name of a spoken language, for the translation prompt. */
export function spokenLanguageName(language: SpokenLanguage): string {
return SPOKEN_LANGUAGES[language];
}
export const translateShowOriginalSetting = new ConfigSettingBoolean({
id: "translate-show-original",
label: "Show original",
+32 -23
View File
@@ -1,6 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
@@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol";
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
import { translateShowOriginalSetting } from "./translate-settings";
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
const NO_MODEL_NOTICE = "No language model is set up.";
/**
* Live translation view. Continuous mic capture is owned by the app wrapper
* (createTranslateAppWindow); this layer feeds finalized speech to the
* assistant's model one stretch at a time, shows the English as it streams
* in, and saves the session to Downloads from the app menu.
* (createTranslateAppWindow); this layer feeds speech to the assistant's
* model one block at a time (provisionally while a block is still being
* spoken), shows the English as it streams in, and saves the session to
* Downloads from the app menu.
*/
export class TranslateLayer implements Layer {
private status = "Starting microphone...";
@@ -30,7 +32,8 @@ export class TranslateLayer implements Layer {
private problem: string | null = null;
private readonly model = new TranslationModel();
private readonly scroll = new TranscriptScroll();
private activeJob: TranslationJob | null = null;
private readonly meter = new MicLevelMeter();
// The request in flight (the model tracks which job it is).
private activeHandle: LlmStreamHandle | null = null;
private requestRender: () => void = () => {};
private removed = false;
@@ -40,10 +43,16 @@ export class TranslateLayer implements Layer {
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
constructor(private readonly stamp: string, private readonly retry: () => void) {}
start(requestRender: () => void): void {
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
start(requestRender: () => void, isVisible: () => boolean): void {
this.requestRender = requestRender;
this.meter.start(requestRender, isVisible);
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.model.accept(event);
if (this.model.accept(event)) {
// The request in flight translates text the transcript has replaced.
this.activeHandle?.cancel();
this.activeHandle = null;
}
this.pump();
requestRender();
});
@@ -68,9 +77,9 @@ export class TranslateLayer implements Layer {
const measure = (line: string) => font.measureText(line);
const maxWidth = width - 64;
image.drawText(font, 24, 20, "Translate", 200);
const status = this.problem ? "Setup needed" : this.translationError || this.status;
image.drawText(font, 24, 40, this.saveNotice || status, 110);
const meter = this.listening && !this.problem ? this.meter : null;
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
let lines: TranslationLine[];
if (this.problem) {
@@ -84,7 +93,6 @@ export class TranslateLayer implements Layer {
}
const step = lineStep(font) + 2;
const bodyTop = 62;
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
const firstLine = this.scroll.layout(lines.length, bodyLines);
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
@@ -119,7 +127,7 @@ export class TranslateLayer implements Layer {
onRemoved(): void {
this.removed = true;
this.activeJob = null;
this.meter.stop();
this.activeHandle?.cancel();
this.activeHandle = null;
this.unsubscribeTranscript?.();
@@ -139,7 +147,10 @@ export class TranslateLayer implements Layer {
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
}
/** Start translating the next waiting stretch of speech, unless one is in flight. */
/**
* Start the next translation request (a final block, else the live block's
* provisional one), unless one is in flight.
*/
private pump(): void {
if (this.removed) return;
const job = this.model.nextJob();
@@ -147,36 +158,34 @@ export class TranslateLayer implements Layer {
const llm = resolveTranslationModel();
if (!llm) {
// The model went away mid-session (a key was cleared): keep the original.
this.model.fail(job.segment);
this.model.fail(job);
this.translationError = NO_MODEL_NOTICE;
this.pump();
return;
}
this.activeJob = job;
const language = resolveTranslateLanguage();
const handle = streamTranslation(llm, {
speech: job.segment.source,
speech: job.source,
context: job.context,
language: language ? spokenLanguageName(language) : "",
onProgress: (textSoFar) => {
if (this.activeJob !== job) return;
this.model.progress(job.segment, textSoFar);
this.requestRender();
if (this.model.progress(job, textSoFar)) this.requestRender();
},
onDone: (translation) => this.settle(job, () => {
this.model.complete(job.segment, translation);
this.model.complete(job, translation);
this.translationError = "";
}),
onError: (message) => this.settle(job, () => {
this.model.fail(job.segment);
this.model.fail(job);
this.translationError = `Translation failed: ${message}`;
}),
});
// A provider that fails synchronously has already settled (and moved on).
if (this.activeJob === job) this.activeHandle = handle;
if (this.model.isActive(job)) this.activeHandle = handle;
}
private settle(job: TranslationJob, apply: () => void): void {
if (this.activeJob !== job) return;
this.activeJob = null;
if (!this.model.isActive(job)) return;
this.activeHandle = null;
apply();
this.pump();
+128 -47
View File
@@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model";
type TranscriptEvent = { text: string; isFinal: boolean };
export type TranslationState = "pending" | "translating" | "done" | "failed";
/** One stretch of finalized speech and its English translation. */
/**
* One block of speech (up to a pause) and its English translation. The last
* block may still be live, its transcript revising; it is translated
* provisionally as it grows, then again once final if its text changed.
*/
export type TranslationSegment = {
source: string;
/** The translation so far (streamed while translating). */
/** Committed by the transcriber; a live block's source still revises. */
final: boolean;
/** The latest finished translation, of `translatedSource`. */
translation: string;
state: TranslationState;
/** What `translation` translates: null until one finishes, behind `source` while provisional. */
translatedSource: string | null;
/** The text streamed so far by the request in flight for this block, else null. */
streaming: string | null;
/** A source whose translation failed; not retried until the source changes. */
failedSource: string | null;
};
export type TranslationTurn = { source: string; translation: string };
/** A segment handed to the model, with the finished turns before it. */
/** A request: a block's source as sent, with the finished turns before it. */
export type TranslationJob = {
segment: TranslationSegment;
/** The source sent (a live block's moves on while the request is in flight). */
source: string;
/** Earlier translated turns, oldest first. */
context: TranslationTurn[];
};
@@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number };
// over between requests; bounded so a long session's requests stay small.
const CONTEXT_TURNS = 4;
const CONTEXT_CHARS = 800;
// A live block is translated again once its text has changed length by this
// much since the last request, so a trickle of revisions (punctuation, one
// corrected word) doesn't spend a request each. A final is always redone.
const PROVISIONAL_MIN_CHANGE = 3;
const TRANSLATION_GRAY = 230;
const PROVISIONAL_GRAY = 170;
const ORIGINAL_GRAY = 120;
const LIVE_GRAY = 90;
function newSegment(source: string, final: boolean): TranslationSegment {
return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null };
}
/** Whether a block's current source still needs a translation request. */
function wantsTranslation(segment: TranslationSegment): boolean {
const { source, translatedSource } = segment;
if (!source || source === translatedSource || source === segment.failedSource) return false;
return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE;
}
/**
* The Translate app's conversation: finalized speech segments with their
* translations, plus the live partial of the utterance in progress.
* The Translate app's conversation: blocks of speech with their translations.
* Translation is serial (one request at a time, in order, so each can carry
* the turns before it as context); finals that arrive while a request is in
* flight merge into one pending segment, so a slow model batches the backlog
* instead of falling further behind one request per pause.
* the turns before it as context): final blocks first, then the live one.
* Finals that arrive while a request is in flight merge into one waiting
* block, so a slow model batches the backlog instead of falling further
* behind one request per pause.
*/
export class TranslationModel {
readonly segments: TranslationSegment[] = [];
/** The utterance in progress (replace semantics), not yet final. */
live = "";
private active: TranslationJob | null = null;
accept(event: TranscriptEvent): void {
/**
* Take a transcript event (replace semantics within the live block).
* Returns true when it made the request in flight moot (its block was
* finalized as other text, or dropped); that request is abandoned here, and
* the caller should cancel it.
*/
accept(event: TranscriptEvent): boolean {
const text = event.text.trim();
if (!event.isFinal) {
this.live = text;
return;
}
const finalText = text || this.live;
this.live = "";
if (!finalText) return;
const last = this.segments[this.segments.length - 1];
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
else this.segments.push({ source: finalText, translation: "", state: "pending" });
const live = last && !last.final ? last : null;
if (!event.isFinal) {
if (live) live.source = text;
else if (text) this.segments.push(newSegment(text, false));
return false;
}
const finalText = text || live?.source || "";
if (!live) {
if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true));
return false;
}
const moot = this.active?.segment === live && this.active.source !== finalText;
if (moot) this.abandonActive();
if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) {
this.segments.pop();
return moot;
}
live.source = finalText;
live.final = true;
return moot;
}
/** Take the next segment to translate, or null while one is in flight or none waits. */
/** Take the next request to make, or null while one is in flight or nothing waits. */
nextJob(): TranslationJob | null {
if (this.segments.some((segment) => segment.state === "translating")) return null;
const segment = this.segments.find((candidate) => candidate.state === "pending");
if (this.active) return null;
const segment = this.segments.find((candidate) => wantsTranslation(candidate));
if (!segment) return null;
segment.state = "translating";
return { segment, context: this.contextBefore(segment) };
segment.streaming = "";
this.active = { segment, source: segment.source, context: this.contextBefore(segment) };
return this.active;
}
progress(segment: TranslationSegment, textSoFar: string): void {
if (segment.state === "translating") segment.translation = textSoFar;
isActive(job: TranslationJob): boolean {
return this.active === job;
}
complete(segment: TranslationSegment, translation: string): void {
segment.translation = translation;
segment.state = "done";
/** Streamed text for the request in flight; false when `job` isn't it any more. */
progress(job: TranslationJob, textSoFar: string): boolean {
if (this.active !== job) return false;
job.segment.streaming = textSoFar;
return true;
}
fail(segment: TranslationSegment): void {
segment.translation = "";
segment.state = "failed";
complete(job: TranslationJob, translation: string): void {
if (this.active !== job) return;
this.active = null;
job.segment.streaming = null;
job.segment.translation = translation;
job.segment.translatedSource = job.source;
}
fail(job: TranslationJob): void {
if (this.active !== job) return;
this.active = null;
job.segment.streaming = null;
job.segment.failedSource = job.source;
}
private abandonActive(): void {
if (this.active) this.active.segment.streaming = null;
this.active = null;
}
/**
* Append a final to the block before `before` (default: the end) when that
* block is final but still waiting for its first request. Returns whether
* it did.
*/
private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean {
const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1;
const waiting = this.segments[index];
if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false;
if (this.active?.segment === waiting) return false;
waiting.source = `${waiting.source} ${finalText}`;
return true;
}
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
@@ -87,7 +162,7 @@ export class TranslationModel {
let chars = 0;
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
const earlier = this.segments[index]!;
if (earlier.state !== "done" || !earlier.translation) continue;
if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue;
chars += earlier.source.length + earlier.translation.length;
if (turns.length && chars > CONTEXT_CHARS) break;
turns.unshift({ source: earlier.source, translation: earlier.translation });
@@ -96,9 +171,11 @@ export class TranslationModel {
}
/**
* Display lines: each segment's translation, bright; its original, dim,
* above it when `showOriginal` is on and in its place until a translation
* arrives (or when it failed); then the live partial, dimmer.
* Display lines: each block's translation, bright once settled (final, and
* of its final text) and dimmer while provisional; its original, dim, above
* it when `showOriginal` is on and in its place until a translation
* arrives. A redo streams in over the translation it replaces only once it
* is as long, so the text doesn't shrink and regrow.
*/
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
const lines: TranslationLine[] = [];
@@ -106,21 +183,25 @@ export class TranslationModel {
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
};
for (const segment of this.segments) {
const translation = segment.translation.trim();
// A finished, empty translation is noise the model chose to drop.
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
if (translation) push(translation, TRANSLATION_GRAY);
if (!segment.source) continue;
const settled = segment.final && segment.translatedSource === segment.source;
const finished = segment.translation.trim();
const streaming = segment.streaming?.trim() ?? "";
const translation = streaming.length >= finished.length ? streaming : finished;
// A settled, empty translation is noise the model chose to drop.
if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY);
if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY);
}
if (this.live) push(this.live, LIVE_GRAY);
return lines;
}
/** Plain-text transcript for saving: each original, then its translation. */
/** Plain-text transcript for saving: each final original, then its translation. */
savedText(): string {
return this.segments
.filter((segment) => segment.final)
.map((segment) => {
const translation = segment.translation.trim();
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`;
if (!translation || translation === segment.source.trim()) return segment.source;
return `${segment.source}\n→ ${translation}`;
})
+12 -11
View File
@@ -74,9 +74,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -2116,10 +2116,10 @@ class DashboardController {
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
}
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
private multilingualProvider(): VoiceProvider | undefined {
/** The provider and language a continuous-capture window needs (Translate's), if any. */
private transcriptionChoice(): TranscriptionChoice | undefined {
for (const options of this.continuousClients.values()) {
if (options.multilingualProvider) return options.multilingualProvider;
if (options.transcription) return options.transcription;
}
return undefined;
}
@@ -2144,20 +2144,21 @@ class DashboardController {
* stands in for the G2 mic.
*/
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
// A window that needs a multilingual provider sets it for every capture
// while it is open, so push-to-talk still shares its stream.
const multilingual = this.multilingualProvider();
// A window that needs its own provider and language sets them for every
// capture while it is open, so push-to-talk still shares its stream.
// Otherwise only Whisper server is told a language (its own setting).
const choice = this.transcriptionChoice();
const provider = choice?.provider ?? voiceProviderSetting.get();
return {
communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null,
provider: multilingual ?? voiceProviderSetting.get(),
provider,
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(),
// Whisper detects the language itself when the speech may be in any.
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
// "My voice only" (Microphones app): verify command utterances
+12 -9
View File
@@ -149,11 +149,12 @@ export class IosPreviewController {
private logLines: string[] = []
private logTimer: ReturnType<typeof setTimeout> | null = null
// Windows that want continuous capture, and what each needs of it (a
// recording, a multilingual provider). They share one capture, which runs
// recording, a provider and language). They share one capture, which runs
// while any of them wants it.
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
// transcriptionKey of the running continuous capture's provider; one that no
// longer matches what its windows need is drained and restarted.
// transcriptionKey of the running continuous capture (provider and
// language); one that no longer matches what its windows need is drained
// and restarted.
private continuousTranscription = ''
// Continuous capture should (re)start once audio is available: a window
// asked, the glasses connected or disconnected, or the glasses lock or the
@@ -1058,15 +1059,17 @@ export class IosPreviewController {
}
/**
* The transcription provider setting and the keys the cloud ones need. A
* window that needs a multilingual provider (Translate) sets it for every
* capture while it is open, with Whisper left to detect the language.
* window that needs its own provider and language (Translate) sets them for
* every capture while it is open; otherwise only Whisper server is told a
* language (its own setting).
*/
private voiceProvider(): VoiceProviderSelection {
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
const provider = choice?.provider ?? voiceProviderSetting.get()
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
whisperServerApiKey: whisperServerApiKeySetting.get() }
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
+11 -10
View File
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
/**
* Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Cloud providers only.
*/
language?: string;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
language: selection.language ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
}
/**
* What a capture transcribes with: the provider, and for Whisper server the
* language it is told to expect. A running capture is shared only by holders
* asking for the same.
* What a capture transcribes with: the provider and the language it is told
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
return selection.provider === "whisper-server"
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
: selection.provider;
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
@@ -105,7 +106,7 @@ export function createCloudSttClient(
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
{ ...options, apiKey, language: (selection.language ?? "").trim() },
continuous,
provider.finalTimeoutMs,
);
+2
View File
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
export type CloudSttOptions = {
apiKey: string;
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
language?: string;
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
+4 -1
View File
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
start(): void {
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
// Without language_code the model detects the language, which can go
// badly wrong on short or noisy segments (Japanese heard as Russian).
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
const listener = {
onOpen: () => {
if (this.closed) return;
+3 -3
View File
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
function sessionConfig(): object {
function sessionConfig(language: string): object {
return {
type: "session.update",
session: {
@@ -209,7 +209,7 @@ function sessionConfig(): object {
audio: {
input: {
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
transcription: { model: MODEL_ID },
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
// Push-to-talk owns the utterance boundary; no server VAD.
turn_detection: null,
},
+2
View File
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
// A hint, not a restriction: other languages are still recognized.
...(this.options.language ? { language_hints: [this.options.language] } : {}),
}),
);
if (this.closed) return;
+5
View File
@@ -3,6 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { WavRecorder } from './wav-recorder.ios'
import { pcmLevel } from '../util/audio-level'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
@@ -24,6 +25,7 @@ export class IosVoiceControlBridge {
private readonly statuses = new Set<(state: VoiceControlState) => void>()
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
private readonly ends = new Set<() => void>()
private readonly levels = new Set<(level: number) => void>()
private status = 'Voice input ready.'
private generation = 0
private session: IosMicrophoneSession | null = null
@@ -60,6 +62,8 @@ export class IosVoiceControlBridge {
}
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} }
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
private ensureNative(): any {
if (!this.native) {
@@ -69,6 +73,7 @@ export class IosVoiceControlBridge {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
this.cloud?.acceptPcm(bytes)
if (this.continuous) this.record(bytes)
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
}
}
return this.native
+18 -1
View File
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
declare const com: any;
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
// stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private readonly micLevelListeners = new Set<(level: number) => void>();
private rawActive = false;
// Who wants the raw tap. It runs while any owner does, so one owner letting
// go (an EvenHub app backgrounding) doesn't silence another.
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
return () => this.rawPcmListeners.delete(listener);
}
/**
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
* each decoded audio chunk, about every 50 ms, from any capture: STT in
* every mode, or the raw tap.
*/
onMicLevel(listener: (level: number) => void): () => void {
this.micLevelListeners.add(listener);
return () => this.micLevelListeners.delete(listener);
}
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
onFrameMeta(listener: FrameMetaListener): () => void {
this.frameMetaListeners.add(listener);
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
if (this.micLevelListeners.size > 0) {
const level = pcmLevel(bytes);
this.micLevelListeners.forEach((listener) => listener(level));
}
},
onFrameMeta: (angleDegrees: number, ssr: number) => {
if (this.frameMetaListeners.size === 0) return;
+7 -2
View File
@@ -85,15 +85,20 @@ export const TRANSLATE_SYSTEM_PROMPT =
"Reply with the English translation only, with no preamble, quotes, notes, or transliteration. " +
"If the speech is empty or meaningless noise, reply with nothing.";
/** User message for the Translate app: earlier turns for context, then the speech to translate. */
/**
* User message for the Translate app: earlier turns for context, then the
* speech to translate. `language` names the spoken language when the user set
* one (the transcription was told to expect it), else "".
*/
export function buildTranslateUserMessage(
speech: string,
context: ReadonlyArray<{ source: string; translation: string }>,
language = "",
): string {
const earlier = context.length
? "Earlier in this conversation (already translated; context only):\n" +
context.map((turn) => `<speech>${turn.source}</speech>\n<english>${turn.translation}</english>`).join("\n") +
"\n\n"
: "";
return `${earlier}Translate this speech into English:\n<speech>${speech}</speech>`;
return `${earlier}Translate this ${language ? `${language} ` : ""}speech into English:\n<speech>${speech}</speech>`;
}
+7 -5
View File
@@ -14,14 +14,16 @@ export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */
recordAs?: string;
/**
* Transcribe with this provider instead of the Transcription Provider
* setting, letting it detect the spoken language (Translate). While any
* client asks for one, every capture uses it, and a shared capture running
* another provider restarts on it.
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
* or "" for the provider to detect it), instead of the Transcription
* Provider setting (Translate). While any client asks for one, every
* capture uses it, and a shared capture running another restarts on it.
*/
multilingualProvider?: VoiceProvider;
transcription?: TranscriptionChoice;
};
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
+92
View File
@@ -0,0 +1,92 @@
import { type GrayImage, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control";
import { lineStep } from "./metrics";
const HEADER_X = 24;
const HEADER_Y = 20;
const TITLE_GRAY = 200;
const STATUS_GRAY = 110;
const SEGMENTS = 8;
const SEGMENT_WIDTH = 4;
const SEGMENT_GAP = 2;
const METER_GAP = 10;
const LIT_GRAY = 200;
const UNLIT_GRAY = 45;
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
const TICK_MS = 200;
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
const DECAY = 0.5;
/**
* A compact segmented mic-level meter for listening views. Levels arrive per
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
* asks for a repaint only when the number of lit segments changed and the
* window is on screen.
*/
export class MicLevelMeter {
readonly width = SEGMENTS * (SEGMENT_WIDTH + SEGMENT_GAP) - SEGMENT_GAP;
private peak = 0;
private shown = 0;
private lit = 0;
private timer: ReturnType<typeof setInterval> | null = null;
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void, isVisible: () => boolean): void {
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
this.peak = Math.max(this.peak, level);
});
this.timer = setInterval(() => {
this.shown = Math.max(this.peak, this.shown * DECAY);
this.peak = 0;
const lit = Math.min(SEGMENTS, Math.round(this.shown * SEGMENTS));
if (lit === this.lit) return;
this.lit = lit;
if (isVisible()) requestRender();
}, TICK_MS);
}
stop(): void {
this.unsubscribe?.();
this.unsubscribe = null;
if (this.timer !== null) clearInterval(this.timer);
this.timer = null;
}
/** Draw the segments with their bottoms on `baseline`, `height` tall. */
paint(image: GrayImage, x: number, baseline: number, height: number): void {
for (let index = 0; index < SEGMENTS; index++) {
const segmentX = x + index * (SEGMENT_WIDTH + SEGMENT_GAP);
image.fillRect(segmentX, baseline - height, SEGMENT_WIDTH, height, index < this.lit ? LIT_GRAY : UNLIT_GRAY);
}
}
}
/**
* The one-line header of the listening views (Transcribe, Translate):
* "Title · status", the status truncated to fit, then the mic level meter at
* the line's end when one is given (while listening). Returns the y at which
* the view's body starts.
*/
export function paintListeningHeader(
image: GrayImage,
font: UiFont,
title: string,
status: string,
meter: MicLevelMeter | null,
): number {
let textRight = image.width - HEADER_X;
if (meter) {
const meterX = textRight - meter.width;
// Roughly cap height, sitting on the text's baseline.
meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6)));
textRight = meterX - METER_GAP;
}
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
if (status) {
const statusX = HEADER_X + Math.ceil(font.measureText(title));
image.drawText(font, statusX, HEADER_Y, truncateText(font, ` · ${status}`, textRight - statusX), STATUS_GRAY);
}
return HEADER_Y + lineStep(font) + 2;
}
+24
View File
@@ -0,0 +1,24 @@
/**
* Mic level-meter math shared by the Microphones app's meters and the
* listening header (Transcribe, Translate): RMS mapped onto a 60 dB meter.
*/
/** Meter position in [0,1] for a chunk of 16-bit little-endian PCM. */
export function pcmLevel(pcm: Uint8Array): number {
let sumSquares = 0;
const count = pcm.length / 2;
if (count === 0) return 0;
for (let i = 0; i < count; i++) {
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
if (value >= 0x8000) value -= 0x10000;
sumSquares += value * value;
}
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
}
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
export function rmsToLevel(rms: number): number {
if (rms <= 0) return 0;
const db = 20 * Math.log10(rms);
return Math.max(0, Math.min(1, (db + 60) / 60));
}
+25 -3
View File
@@ -2,7 +2,7 @@ const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function harness(provider = 'elevenlabs', continuous = true) {
function harness(provider = 'elevenlabs', continuous = true, extraOptions = {}) {
const timers = new Map();
let timerId = 0;
const sockets = [];
@@ -30,7 +30,7 @@ function harness(provider = 'elevenlabs', continuous = true) {
const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]];
const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts');
const client = new ReconnectingSttClient(options => new Provider(options), {
apiKey: 'test', onTranscript: event => events.push(event),
apiKey: 'test', ...extraOptions, onTranscript: event => events.push(event),
onStatus: status => statuses.push(status), onError: error => errors.push(error),
}, () => continuous);
client.start();
@@ -203,7 +203,10 @@ test('provider selection falls back to on-device without a key and connects the
FaceclawWebSocket: class { constructor(url, _listener, header, value) { sockets.push({ url, header, value }); } },
} } },
}, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } });
const { createCloudSttClient, usesCloudStt } = load('app/native/cloud-stt-provider.ts');
const { createCloudSttClient, usesCloudStt, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
// A running capture is shared only on the same provider and language.
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
const statuses = [];
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
@@ -217,4 +220,23 @@ test('provider selection falls back to on-device without a key and connects the
createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: ' xi ' }, options, () => false).start();
assert.deepEqual(sockets.map(({ url, header, value }) => [new URL(url).host, header, value]),
[['stt-rt.soniox.com', null, null], ['api.elevenlabs.io', 'xi-api-key', 'xi']]);
createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: 'xi', language: ' ja ' }, options, () => false).start();
assert.equal(new URL(sockets.at(-1).url).searchParams.get('language_code'), 'ja');
});
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
for (const language of ['ja', '']) {
const elevenlabs = harness('elevenlabs', true, { language });
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
const soniox = harness('soniox', true, { language });
soniox.open();
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
const openai = harness('openai', true, { language });
openai.open();
const transcription = JSON.parse(openai.sockets[0].sent[0]).session.audio.input.transcription;
assert.equal(transcription.model, 'gpt-realtime-whisper');
assert.equal(transcription.language, language || undefined);
}
});
+9 -6
View File
@@ -348,7 +348,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
get continuousActive() { return !!this.source || this.draining; },
prepare: async () => true,
startGlassesCapture: async (_session, voice, _log, _endpointing, continuous) => {
calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.whisperServerLanguage}`); bridge.source = 'glasses'; },
calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.language}`); bridge.source = 'glasses'; },
startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; },
drain() { if (this.source) { this.source = null; this.draining = true; } },
stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); },
@@ -361,7 +361,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
: id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.whisperServerLanguage}` } : {},
: id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.language}` } : {},
});
const host = Object.create(IosPreviewController.prototype);
Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true,
@@ -381,14 +381,17 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
// Translate needs a multilingual provider: the shared capture drains and
// restarts on it (Whisper left to detect the language), and moves back to
// the main provider once Translate closes.
calls.length = 0; host.startContinuousVoiceCapture('translate', { multilingualProvider: 'soniox' }); await settle();
// Translate needs its own provider and language: the shared capture drains
// and restarts on them, again when only the language changes, and moves
// back to the main provider (and Whisper's own language) once Translate closes.
calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: '' } }); await settle();
assert.deepEqual(calls, [['stop']]);
drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
assert.deepEqual(providers.slice(-2), ['whisper-server:en', 'soniox:']);
calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: 'ja' } }); drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
assert.equal(providers.at(-1), 'soniox:ja');
calls.length = 0; host.stopContinuousVoiceCapture('translate'); drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
assert.equal(providers.at(-1), 'whisper-server:en');
+100
View File
@@ -0,0 +1,100 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const { pcmLevel } = loader()('app/util/audio-level.ts');
function harness() {
const levelListeners = new Set();
let tick = null;
const { MicLevelMeter, paintListeningHeader } = loader({
setInterval(fn) { tick = fn; return 1; },
clearInterval() { tick = null; },
}, {
'../native/voice-control': { voiceControlBridge: {
onMicLevel(listener) { levelListeners.add(listener); return () => levelListeners.delete(listener); },
} },
})('app/ui/listening-header.ts');
const level = value => { for (const listener of levelListeners) listener(value); };
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
}
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
function fakeImage(width = 576) {
const texts = [], rects = [];
return { width, texts, rects,
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
}
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
function pcm(amplitude, samples = 800) {
const bytes = new Uint8Array(samples * 2);
for (let i = 0; i < samples; i++) {
const value = (i % 2 ? amplitude : -amplitude) & 0xffff;
bytes[i * 2] = value & 255;
bytes[i * 2 + 1] = value >> 8;
}
return bytes;
}
test('pcm level maps RMS onto a 60 dB meter', () => {
assert.equal(pcmLevel(new Uint8Array(0)), 0);
assert.equal(pcmLevel(pcm(0)), 0);
assert.ok(Math.abs(pcmLevel(pcm(3277)) - 2 / 3) < 0.01, '-20 dBFS sits two thirds up');
assert.ok(pcmLevel(pcm(32767)) > 0.99);
});
test('the meter repaints only when the lit segment count changes, and only while visible', () => {
const h = harness();
const meter = new h.MicLevelMeter();
let renders = 0, visible = true;
meter.start(() => renders++, () => visible);
h.level(0.3); h.level(0.5); h.level(0.2);
h.tick();
assert.equal(renders, 1, 'the peak of the tick lights 4 of 8');
h.level(0.5); h.tick();
assert.equal(renders, 1, 'same segment count, no repaint');
// Off screen the level still tracks, so the next visible paint is current.
visible = false;
h.level(1); h.tick();
assert.equal(renders, 1);
const image = fakeImage();
meter.paint(image, 0, 10, 6);
assert.equal(image.rects.filter(rect => rect.gray === 200).length, 8);
// Silence decays the meter to empty within about a second.
visible = true;
for (let i = 0; i < 5; i++) h.tick();
const after = fakeImage();
meter.paint(after, 0, 10, 6);
assert.equal(after.rects.filter(rect => rect.gray === 200).length, 0);
assert.ok(renders > 1);
meter.stop();
assert.equal(h.ticking(), false);
assert.equal(h.levelListeners.size, 0);
});
test('the header puts title and status on one line, truncated before the meter', () => {
const h = harness();
const meter = new h.MicLevelMeter();
const image = fakeImage(300);
const bodyTop = h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter);
assert.equal(bodyTop, 20 + 18 + 2);
assert.deepEqual(image.texts.map(({ x, y, gray }) => [x, y, gray]), [[24, 20, 200], [24 + 54, 20, 110]]);
assert.equal(image.texts[0].text, 'Translate');
const status = image.texts[1].text;
assert.ok(status.startsWith(' · Listening') && status.endsWith('...'), status);
const meterX = Math.min(...image.rects.map(rect => rect.x));
assert.equal(meterX, 300 - 24 - meter.width);
assert.ok(24 + 54 + status.length * 6 <= meterX - 10, 'status stops short of the meter');
// Segments sit on the text baseline.
assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent));
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null);
assert.equal(idle.rects.length, 0);
assert.equal(idle.texts[1].text, ' · Starting microphone...');
});
+109 -23
View File
@@ -32,14 +32,13 @@ function settingsHarness({ main = 'onboard', setUp = [] } = {}) {
})('app/apps/translate/translate-settings.ts');
}
test('finals become segments; finals arriving mid-translation merge into one pending segment', () => {
test('finals translate in order; finals arriving mid-request merge into one waiting block', () => {
const model = new TranslationModel();
model.accept({ text: 'Hola, ¿qué', isFinal: false });
assert.equal(model.live, 'Hola, ¿qué');
model.accept({ text: 'Hola, ¿qué tal?', isFinal: true });
assert.equal(model.live, '');
assert.equal(model.segments.length, 1);
const first = model.nextJob();
assert.equal(first.segment.source, 'Hola, ¿qué tal?');
assert.equal(first.source, 'Hola, ¿qué tal?');
assert.deepEqual(plain(first.context), []);
assert.equal(model.nextJob(), null, 'one request at a time');
@@ -49,30 +48,91 @@ test('finals become segments; finals arriving mid-translation merge into one pen
assert.equal(model.segments.length, 2);
assert.equal(model.segments[1].source, 'Muy bien. Gracias.');
model.progress(first.segment, 'Hi, how');
assert.equal(first.segment.translation, 'Hi, how');
model.complete(first.segment, 'Hi, how are you?');
assert.equal(model.progress(first, 'Hi, how'), true);
assert.equal(first.segment.streaming, 'Hi, how');
model.complete(first, 'Hi, how are you?');
const second = model.nextJob();
assert.equal(second.segment.source, 'Muy bien. Gracias.');
assert.equal(second.source, 'Muy bien. Gracias.');
assert.deepEqual(plain(second.context), [{ source: 'Hola, ¿qué tal?', translation: 'Hi, how are you?' }]);
model.complete(second.segment, 'Very well. Thanks.');
model.complete(second, 'Very well. Thanks.');
assert.equal(model.nextJob(), null);
});
test('the live block is translated provisionally, redone as it grows, and settles on a matching final', () => {
const model = new TranslationModel();
model.accept({ text: '今日は', isFinal: false });
const first = model.nextJob();
assert.equal(first.source, '今日は');
model.accept({ text: '今日はいい天気', isFinal: false }); // grows while the request is in flight
assert.equal(model.nextJob(), null);
model.complete(first, 'Today');
assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'Today', gray: 170 }]);
const second = model.nextJob();
assert.equal(second.source, '今日はいい天気');
// The redo streams over the old text only once it is as long.
model.progress(second, 'To');
assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Today']);
model.progress(second, 'The weather');
assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['The weather']);
model.complete(second, 'The weather is nice today');
// A final matching what was translated needs no further request.
model.accept({ text: '今日はいい天気', isFinal: true });
assert.equal(model.nextJob(), null);
assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'The weather is nice today', gray: 230 }]);
});
test('a final that differs from the request in flight makes it moot; a matching one keeps it', () => {
const model = new TranslationModel();
model.accept({ text: 'Bonjour', isFinal: false });
const stale = model.nextJob();
assert.equal(model.accept({ text: 'Bonjour à tous', isFinal: true }), true);
assert.equal(model.isActive(stale), false);
assert.equal(model.progress(stale, 'Hello'), false);
const redo = model.nextJob();
assert.equal(redo.source, 'Bonjour à tous');
model.complete(stale, 'Hello'); // a late reply from the cancelled request is ignored
assert.equal(model.isActive(redo), true);
model.complete(redo, 'Hello everyone');
model.accept({ text: 'Merci', isFinal: false });
const kept = model.nextJob();
assert.equal(model.accept({ text: 'Merci', isFinal: true }), false);
model.complete(kept, 'Thank you');
assert.equal(model.nextJob(), null);
assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.gray)), [230, 230]);
});
test('small revisions of a live block wait for a bigger change; its final is always redone', () => {
const model = new TranslationModel();
model.accept({ text: 'abcdef', isFinal: false });
model.complete(model.nextJob(), 'one');
model.accept({ text: 'abcdeg', isFinal: false });
assert.equal(model.nextJob(), null);
model.accept({ text: 'abcdefgh', isFinal: false });
assert.equal(model.nextJob(), null);
model.accept({ text: 'abcdefghi', isFinal: false });
model.complete(model.nextJob(), 'two');
model.accept({ text: 'abcdefghj', isFinal: true });
assert.equal(model.nextJob().source, 'abcdefghj');
});
test('an empty final keeps the live partial it replaces', () => {
const model = new TranslationModel();
model.accept({ text: 'Bonjour', isFinal: false });
model.accept({ text: '', isFinal: true });
assert.equal(model.segments[0].source, 'Bonjour');
assert.equal(model.segments[0].final, true);
});
test('context skips failed and dropped turns and stays bounded', () => {
test('context skips failed, dropped and provisional turns and stays bounded', () => {
const model = new TranslationModel();
const translate = (source, translation) => {
model.accept({ text: source, isFinal: true });
const job = model.nextJob();
if (translation === null) model.fail(job.segment);
else model.complete(job.segment, translation);
if (translation === null) model.fail(job);
else model.complete(job, translation);
};
translate('uno', 'one');
translate('ruido', null);
@@ -87,18 +147,29 @@ test('context skips failed and dropped turns and stays bounded', () => {
const long = new TranslationModel();
for (let i = 0; i < 3; i++) {
long.accept({ text: 'x'.repeat(500), isFinal: true });
long.complete(long.nextJob().segment, 'y'.repeat(500));
long.complete(long.nextJob(), 'y'.repeat(500));
}
long.accept({ text: 'next', isFinal: true });
assert.equal(long.nextJob().context.length, 1, 'the newest turn is kept even when it alone is over the cap');
// A final whose redo failed still shows its provisional translation, but
// that translates other text, so it isn't context.
const stale = new TranslationModel();
stale.accept({ text: 'Ich gehe', isFinal: false });
stale.complete(stale.nextJob(), 'I go');
stale.accept({ text: 'Ich gehe nach Hause', isFinal: true });
stale.fail(stale.nextJob());
assert.deepEqual(plain(stale.lines(measure, 100, false)), [{ text: 'I go', gray: 170 }]);
stale.accept({ text: 'Tschüss', isFinal: true });
assert.deepEqual(plain(stale.nextJob().context), []);
});
test('display lines: translations bright, originals dim until translated or when shown, live dimmest', () => {
test('display lines: settled bright, provisional dimmer, originals dim until translated or when shown', () => {
const model = new TranslationModel();
model.accept({ text: 'Guten Morgen', isFinal: true });
model.complete(model.nextJob().segment, 'Good morning');
model.complete(model.nextJob(), 'Good morning');
model.accept({ text: 'Wie geht es?', isFinal: true });
const asking = model.nextJob().segment;
const asking = model.nextJob();
model.accept({ text: 'Danke', isFinal: false });
assert.deepEqual(plain(model.lines(measure, 100, false)), [
{ text: 'Good morning', gray: 230 },
@@ -107,28 +178,29 @@ test('display lines: translations bright, originals dim until translated or when
]);
model.progress(asking, 'How are');
assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Good morning', 'How are', 'Danke']);
assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 230, 90]);
assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 170, 90]);
assert.deepEqual(plain(model.lines(measure, 7, false).slice(0, 2)), [{ text: 'Good', gray: 230 }, { text: 'morning', gray: 230 }]);
// A failed segment keeps its original on screen.
// A failed block keeps its original on screen.
model.fail(asking);
assert.deepEqual(plain(model.lines(measure, 100, false)[1]), { text: 'Wie geht es?', gray: 120 });
// A finished, empty translation is noise: hidden unless originals are shown.
// A settled, empty translation is noise: hidden unless originals are shown.
model.accept({ text: 'mmm', isFinal: true });
model.complete(model.nextJob().segment, '');
model.complete(model.nextJob(), '');
assert.equal(model.lines(measure, 100, false).some(line => line.text === 'mmm'), false);
assert.equal(model.lines(measure, 100, true).some(line => line.text === 'mmm'), true);
});
test('saved text pairs each original with its translation', () => {
test('saved text pairs each final original with its translation', () => {
const model = new TranslationModel();
for (const [source, translation] of [['Ciao', 'Hello'], ['Okay.', 'Okay.'], ['boh', null]]) {
model.accept({ text: source, isFinal: true });
const job = model.nextJob();
if (translation === null) model.fail(job.segment);
else model.complete(job.segment, translation);
if (translation === null) model.fail(job);
else model.complete(job, translation);
}
model.accept({ text: 'ancora', isFinal: false }); // still live: not saved
assert.equal(model.savedText(), 'Ciao\n→ Hello\n\nOkay.\n\nboh\n→ (not translated)');
});
@@ -138,6 +210,8 @@ test('the user message carries earlier turns as context, then the speech', () =>
assert.match(message, /context only/);
assert.match(message, /<speech>Bien<\/speech>\n<english>Fine<\/english>\n\nTranslate this speech/);
assert.ok(message.endsWith('<speech>¿Y tú?</speech>'));
assert.equal(buildTranslateUserMessage('こんにちは', [], 'Japanese'),
'Translate this Japanese speech into English:\n<speech>こんにちは</speech>');
});
test('Auto picks the main provider when it is multilingual, else the first one set up', () => {
@@ -164,3 +238,15 @@ test('an explicit provider is used only while it is set up, and on-device ones a
assert.equal(settings.translateProviderSetting.isDisabled('auto'), false);
assert.deepEqual(plain(settings.translateProviderSetting.values), ['auto', 'elevenlabs', 'whisper', 'soniox', 'whisper-server']);
});
test('the spoken language defaults to auto-detect and resolves to an ISO 639-1 code once chosen', () => {
const settings = settingsHarness();
assert.equal(settings.resolveTranslateLanguage(), '');
assert.equal(settings.translateLanguageSetting.displayValue('auto'), 'Auto-detect');
settings.translateLanguageSetting.set('ja');
assert.equal(settings.resolveTranslateLanguage(), 'ja');
assert.equal(settings.spokenLanguageName('ja'), 'Japanese');
assert.equal(settings.translateLanguageSetting.values[0], 'auto');
assert.ok(settings.translateLanguageSetting.values.slice(1).every(code => /^[a-z]{2}$/.test(code)));
assert.equal(settings.translateLanguageSetting.values.includes('en'), false, 'English is the target, not a source');
});