Add Translate app

This commit is contained in:
James Babcock committed 2026-10-24 16:16:26 -07:00
1 parent 07859c95ad
commit c84d301c78
19 files changed
+926 -57

No files matched your search

+22 -6
View File
@@ -88,7 +88,7 @@ test('iOS host forwards hands-free capture mode to the voice bridge', async () =
? { iosVoiceInput: { startGlassesCapture: (_session, _voice, _log, endpointing) => captures.push(endpointing) } }
: id === '../ui/dashboard-settings' ? voiceSettings : {} });
const host = Object.create(IosPreviewController.prototype);
host.communicator = {}; host.state = { phase: 'connected' };
host.communicator = {}; host.state = { phase: 'connected' }; host.continuousClients = new Map();
await host.startVoiceCapture(true);
await host.startVoiceCapture();
assert.deepEqual(captures, [true, false]);
@@ -327,6 +327,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
});
const host = Object.create(IosPreviewController.prototype);
host.active = true; host.glassesLocked = false; host.communicator = null; host.state = { phase: 'disconnected' };
host.continuousClients = new Map();
assert.equal(await host.prepareVoiceCapture(), true);
await host.startVoiceCapture(true);
assert.deepEqual(calls, [['prepare', true, true, 'onboard'], ['prepare', true, true, 'onboard'], ['phone', 'onboard', true]]);
@@ -339,14 +340,15 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
});
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
const calls = [], claims = [];
const calls = [], claims = [], providers = [];
// Models the bridge's continuous state: listening on a source, or draining after a stop.
const bridge = { source: null, draining: false, recording: '',
supportsContinuous: voice => voice.provider !== 'onboard',
get continuousSource() { return this.source; },
get continuousActive() { return !!this.source || this.draining; },
prepare: async () => true,
startGlassesCapture: async (_session, _voice, _log, _endpointing, continuous) => { calls.push(['glasses', continuous]); bridge.source = 'glasses'; },
startGlassesCapture: async (_session, voice, _log, _endpointing, continuous) => {
calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.whisperServerLanguage}`); bridge.source = 'glasses'; },
startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; },
drain() { if (this.source) { this.source = null; this.draining = true; } },
stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); },
@@ -354,10 +356,12 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
stopPhoneCapture() { if (bridge.source === 'phone') bridge.drain(); },
recordTo: name => { bridge.recording = name; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, saveTranscribeAudioSetting: { get: () => true } };
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
saveTranscribeAudioSetting: { get: () => true } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } : {},
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
: id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.whisperServerLanguage}` } : {},
});
const host = Object.create(IosPreviewController.prototype);
Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true,
@@ -369,7 +373,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
const drained = () => { bridge.draining = false; host.syncContinuousCapture(); };
const session = phase => { host.state = { phase }; if (phase !== 'connected') bridge.handleSessionEnded(); host.syncContinuousSession(); };
host.startContinuousVoiceCapture('transcribe', 'transcribe-1'); await settle();
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
host.startContinuousVoiceCapture('calculator'); await settle();
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
assert.equal(bridge.recording, 'transcribe-1');
@@ -377,6 +381,18 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
// Translate needs a multilingual provider: the shared capture drains and
// restarts on it (Whisper left to detect the language), and moves back to
// the main provider once Translate closes.
calls.length = 0; host.startContinuousVoiceCapture('translate', { multilingualProvider: 'soniox' }); await settle();
assert.deepEqual(calls, [['stop']]);
drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
assert.deepEqual(providers.slice(-2), ['whisper-server:en', 'soniox:']);
calls.length = 0; host.stopContinuousVoiceCapture('translate'); drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
assert.equal(providers.at(-1), 'whisper-server:en');
// A reconnect drains the capture, waits for the glasses, and resumes on them.
calls.length = 0; session('retrying'); drained(); await settle();
assert.deepEqual(calls, []);
+1
View File
@@ -139,6 +139,7 @@ function fixture() {
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {}, onSpeechEnd: () => () => {} } },
'../native/cloud-stt-provider': { transcriptionKey: voice => voice.provider },
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
'../native/preview-display.ios': { PreviewDisplayTarget },
+166
View File
@@ -0,0 +1,166 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const { TranslationModel } = loader()('app/apps/translate/translation-model.ts');
const { buildTranslateUserMessage } = loader()('app/prompts.ts');
// One character per pixel, so wrapping is predictable.
const measure = text => text.length;
// Values built inside the loader's sandbox have its prototypes; compare as plain data.
const plain = value => JSON.parse(JSON.stringify(value));
function settingsHarness({ main = 'onboard', setUp = [] } = {}) {
class Setting {
constructor(options) { Object.assign(this, options); this.value = options.defaultValue; }
get() { return this.value; }
set(value) { this.value = value; return value; }
displayValue(value = this.get()) { return this.formatValue ? this.formatValue(value) : String(value); }
}
const labels = { elevenlabs: 'ElevenLabs', whisper: 'OpenAI (Whisper)', soniox: 'Soniox', 'whisper-server': 'Whisper server' };
const available = new Set(setUp);
return loader({}, {
'../../ui/dashboard-settings': {
ConfigSettingEnum: Setting,
ConfigSettingBoolean: Setting,
voiceProviderSetting: {
get: () => main,
isDisabled: provider => !available.has(provider),
displayValue: provider => labels[provider] ?? provider,
},
},
})('app/apps/translate/translate-settings.ts');
}
test('finals become segments; finals arriving mid-translation merge into one pending segment', () => {
const model = new TranslationModel();
model.accept({ text: 'Hola, ¿qué', isFinal: false });
assert.equal(model.live, 'Hola, ¿qué');
model.accept({ text: 'Hola, ¿qué tal?', isFinal: true });
assert.equal(model.live, '');
const first = model.nextJob();
assert.equal(first.segment.source, 'Hola, ¿qué tal?');
assert.deepEqual(plain(first.context), []);
assert.equal(model.nextJob(), null, 'one request at a time');
model.accept({ text: 'Muy bien.', isFinal: true });
model.accept({ text: '', isFinal: true }); // an empty final with no partial adds nothing
model.accept({ text: 'Gracias.', isFinal: true });
assert.equal(model.segments.length, 2);
assert.equal(model.segments[1].source, 'Muy bien. Gracias.');
model.progress(first.segment, 'Hi, how');
assert.equal(first.segment.translation, 'Hi, how');
model.complete(first.segment, 'Hi, how are you?');
const second = model.nextJob();
assert.equal(second.segment.source, 'Muy bien. Gracias.');
assert.deepEqual(plain(second.context), [{ source: 'Hola, ¿qué tal?', translation: 'Hi, how are you?' }]);
model.complete(second.segment, 'Very well. Thanks.');
assert.equal(model.nextJob(), null);
});
test('an empty final keeps the live partial it replaces', () => {
const model = new TranslationModel();
model.accept({ text: 'Bonjour', isFinal: false });
model.accept({ text: '', isFinal: true });
assert.equal(model.segments[0].source, 'Bonjour');
});
test('context skips failed and dropped turns and stays bounded', () => {
const model = new TranslationModel();
const translate = (source, translation) => {
model.accept({ text: source, isFinal: true });
const job = model.nextJob();
if (translation === null) model.fail(job.segment);
else model.complete(job.segment, translation);
};
translate('uno', 'one');
translate('ruido', null);
translate('mmm', '');
for (let i = 0; i < 6; i++) translate(`frase ${i}`, `sentence ${i}`);
model.accept({ text: 'último', isFinal: true });
const { context } = model.nextJob();
assert.equal(context.length, 4);
assert.deepEqual(plain(context.at(-1)), { source: 'frase 5', translation: 'sentence 5' });
assert.ok(context.every(turn => turn.translation));
const long = new TranslationModel();
for (let i = 0; i < 3; i++) {
long.accept({ text: 'x'.repeat(500), isFinal: true });
long.complete(long.nextJob().segment, 'y'.repeat(500));
}
long.accept({ text: 'next', isFinal: true });
assert.equal(long.nextJob().context.length, 1, 'the newest turn is kept even when it alone is over the cap');
});
test('display lines: translations bright, originals dim until translated or when shown, live dimmest', () => {
const model = new TranslationModel();
model.accept({ text: 'Guten Morgen', isFinal: true });
model.complete(model.nextJob().segment, 'Good morning');
model.accept({ text: 'Wie geht es?', isFinal: true });
const asking = model.nextJob().segment;
model.accept({ text: 'Danke', isFinal: false });
assert.deepEqual(plain(model.lines(measure, 100, false)), [
{ text: 'Good morning', gray: 230 },
{ text: 'Wie geht es?', gray: 120 },
{ text: 'Danke', gray: 90 },
]);
model.progress(asking, 'How are');
assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Good morning', 'How are', 'Danke']);
assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 230, 90]);
assert.deepEqual(plain(model.lines(measure, 7, false).slice(0, 2)), [{ text: 'Good', gray: 230 }, { text: 'morning', gray: 230 }]);
// A failed segment keeps its original on screen.
model.fail(asking);
assert.deepEqual(plain(model.lines(measure, 100, false)[1]), { text: 'Wie geht es?', gray: 120 });
// A finished, empty translation is noise: hidden unless originals are shown.
model.accept({ text: 'mmm', isFinal: true });
model.complete(model.nextJob().segment, '');
assert.equal(model.lines(measure, 100, false).some(line => line.text === 'mmm'), false);
assert.equal(model.lines(measure, 100, true).some(line => line.text === 'mmm'), true);
});
test('saved text pairs each original with its translation', () => {
const model = new TranslationModel();
for (const [source, translation] of [['Ciao', 'Hello'], ['Okay.', 'Okay.'], ['boh', null]]) {
model.accept({ text: source, isFinal: true });
const job = model.nextJob();
if (translation === null) model.fail(job.segment);
else model.complete(job.segment, translation);
}
assert.equal(model.savedText(), 'Ciao\n→ Hello\n\nOkay.\n\nboh\n→ (not translated)');
});
test('the user message carries earlier turns as context, then the speech', () => {
assert.equal(buildTranslateUserMessage('Hola', []), 'Translate this speech into English:\n<speech>Hola</speech>');
const message = buildTranslateUserMessage('¿Y tú?', [{ source: 'Bien', translation: 'Fine' }]);
assert.match(message, /context only/);
assert.match(message, /<speech>Bien<\/speech>\n<english>Fine<\/english>\n\nTranslate this speech/);
assert.ok(message.endsWith('<speech>¿Y tú?</speech>'));
});
test('Auto picks the main provider when it is multilingual, else the first one set up', () => {
let settings = settingsHarness({ main: 'soniox', setUp: ['elevenlabs', 'soniox'] });
assert.equal(settings.resolveTranslateProvider(), 'soniox');
assert.equal(settings.translateProviderSetting.displayValue('auto'), 'Auto (Soniox)');
settings = settingsHarness({ main: 'onboard', setUp: ['soniox', 'whisper'] });
assert.equal(settings.resolveTranslateProvider(), 'whisper');
// The main provider counts only when it is set up.
settings = settingsHarness({ main: 'elevenlabs', setUp: ['soniox'] });
assert.equal(settings.resolveTranslateProvider(), 'soniox');
settings = settingsHarness({ main: 'onboard', setUp: [] });
assert.equal(settings.resolveTranslateProvider(), null);
assert.equal(settings.translateProviderSetting.displayValue('auto'), 'Auto (none set up)');
});
test('an explicit provider is used only while it is set up, and on-device ones are never offered', () => {
const settings = settingsHarness({ main: 'soniox', setUp: ['soniox', 'whisper-server'] });
assert.equal(settings.resolveTranslateProvider('whisper-server'), 'whisper-server');
assert.equal(settings.resolveTranslateProvider('elevenlabs'), null);
assert.equal(settings.translateProviderSetting.isDisabled('elevenlabs'), true);
assert.equal(settings.translateProviderSetting.isDisabled('auto'), false);
assert.deepEqual(plain(settings.translateProviderSetting.values), ['auto', 'elevenlabs', 'whisper', 'soniox', 'whisper-server']);
});
+36 -14
View File
@@ -157,26 +157,30 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
}
});
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
let controller;
class Controller {
constructor() { controller = this; }
setListener(listener) { this.listener = listener; }
setCommunicator() {} setUsePhoneMic() {} setSaveRecordings() {} setEndpointing() {}
setNoiseSuppression() {} setBeamFilter() {} clearSpeakerVerification() {} setOnboardModelKind() {}
start(_mode, id) { this.captureId = id; this.active = true; }
stop() { this.active = false; }
isCapturing() { return this.active; }
}
/** The Android voice bridge over a fake native controller (on-device recognition only). */
function androidBridge() {
const controller = {
starts: 0,
setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; },
isCapturing() { return this.active; },
};
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null },
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: Controller,
FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; },
} } } });
const bridge = new FaceclawVoiceControlBridge();
return { bridge: new FaceclawVoiceControlBridge(), controller };
}
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
const { bridge, controller } = androidBridge();
const options = { provider: 'onboard-whisper' };
const events = [];
bridge.onTranscript(event => events.push(event.text));
@@ -193,3 +197,21 @@ test('native completion is capture-specific, follows the final, and does not wai
assert.equal(controller.active, true);
bridge.stopContinuousCapture(); controller.listener.onStopped(controller.captureId);
});
test('a shared capture restarts for a holder that needs another provider, and is shared otherwise', () => {
const { bridge, controller } = androidBridge();
bridge.retargetContinuousCapture({ provider: 'soniox' }); // nothing continuous to retarget
assert.equal(controller.starts, 0);
bridge.startPushToTalk({ provider: 'onboard' });
bridge.startContinuousCapture({ provider: 'onboard' });
assert.equal(controller.starts, 1);
bridge.retargetContinuousCapture({ provider: 'soniox' });
assert.equal(controller.starts, 2);
bridge.retargetContinuousCapture({ provider: 'soniox' });
bridge.startPushToTalk({ provider: 'soniox' });
assert.equal(controller.starts, 2);
// Push-to-talk still holds the restarted stream once continuous lets go.
bridge.stopContinuousCapture();
assert.equal(controller.active, true);
assert.equal(bridge.holdsCapture('ptt'), true);
});