Merge multilingual Whisper transcription

# Conflicts:
#	app/g2/dashboard-controller.ts
This commit is contained in:
Blendman974 committed 2026-11-06 04:31:25 +01:00
commit cb4635d959
21 files changed
+375 -39

No files matched your search

@@ -55,6 +55,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
ASR_MODELS: {
moonshine: { label: 'Moonshine', sizeBytes: 1 },
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
},
asrModelState: () => ({}), cancelAsrModelDownload() {},
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
+1
View File
@@ -26,6 +26,7 @@ function settingsFixture(store = new Map()) {
};
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
}};
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
+4 -2
View File
@@ -207,6 +207,8 @@ test('provider selection falls back to on-device without a key and connects the
// A running capture is shared only on the same provider and language.
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
const statuses = [];
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
@@ -225,13 +227,13 @@ test('provider selection falls back to on-device without a key and connects the
});
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
for (const language of ['ja', '']) {
for (const language of ['fr', 'ja', '']) {
const elevenlabs = harness('elevenlabs', true, { language });
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
const soniox = harness('soniox', true, { language });
soniox.open();
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
const openai = harness('openai', true, { language });
openai.open();
+1
View File
@@ -12,6 +12,7 @@ function settingsFixture(store = new Map()) {
};
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
}};
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
+3 -1
View File
@@ -6,6 +6,7 @@ const ts = require('typescript');
const path = require('node:path');
// The provider settings ios-preview-controller hands the voice bridge.
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
getVoiceTranscriptionLanguage: () => '',
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
@@ -168,6 +169,7 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
let cachedFont = 'Light';
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
setTimeout: fn => tasks.push(fn),
});
@@ -384,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
recordTo: name => { bridge.recording = name; },
setRawSession: session => { bridge.rawSession = session; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
saveTranscribeAudioSetting: { get: () => true } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
@@ -0,0 +1,35 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class WhisperModelFilesTest {
@Test fun existingEnglishFilesRemainUsable() {
for (language in listOf(null, "en", " EN ")) {
val model = WhisperModelFiles.forLanguage(language)
assertTrue(model.englishOnly)
assertEquals("en", model.language)
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
}
}
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
for (language in listOf("fr", "de", "ja", " FR ")) {
val model = WhisperModelFiles.forLanguage(language)
assertFalse(model.englishOnly)
assertEquals(language.trim().lowercase(), model.language)
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
}
}
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
val model = WhisperModelFiles.forLanguage("")
assertFalse(model.englishOnly)
assertEquals("", model.language)
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
}
}
+1
View File
@@ -28,6 +28,7 @@
"../app/ui/menu-core.ts",
"../app/phone-ui/phone-gestures.ts",
"../app/native/java-direct-buffer.ts",
"../app/native/transcription-language.ts",
"../app/native/notification-types.ts",
"../app/g2/firmware-compat.ts",
"../app/g2/glasses-hardware-identity.ts",
+24 -2
View File
@@ -163,7 +163,8 @@ function androidBridge() {
starts: 0,
setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; },
isCapturing() { return this.active; },
@@ -171,7 +172,8 @@ function androidBridge() {
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
'./cloud-stt-provider': { createCloudSttClient: () => null,
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; },
@@ -179,6 +181,26 @@ function androidBridge() {
return { bridge: new FaceclawVoiceControlBridge(), controller };
}
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
const { bridge, controller } = androidBridge();
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
assert.equal(controller.modelKind, 'whisper');
assert.equal(controller.language, 'fr');
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
assert.equal(controller.starts, 1);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
assert.equal(controller.language, '');
assert.equal(controller.starts, 2);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
// An omitted language preserves the native bridge's original English behavior.
assert.equal(controller.language, 'en');
assert.equal(controller.starts, 3);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
assert.equal(controller.language, 'en');
assert.equal(controller.starts, 3);
bridge.stop();
});
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
const { bridge, controller } = androidBridge();
const options = { provider: 'onboard-whisper' };
+131
View File
@@ -0,0 +1,131 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
function load(file, requireModule = () => ({}), globals = {}) {
const exports = {};
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
return exports;
}
const languages = load('app/native/transcription-language.ts');
function settings(values = new Map()) {
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
: name.includes('settings-store') ? {
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
setStringSetting: (key, value) => values.set(key, value),
getBooleanSetting: (_key, fallback) => fallback,
onSettingsStoreChanged() {},
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
}
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
const s = settings(values);
assert.equal(s.voiceLanguageSetting.get(), 'default');
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
});
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
let s = settings(values);
s.voiceLanguageSetting.set('fr');
s = settings(values);
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
}
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
s.voiceLanguageSetting.set('auto');
s = settings(values);
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
s.voiceLanguageSetting.set('default');
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
});
test('corrupt language preferences revert to the compatible provider defaults', () => {
const s = settings(new Map([['voice.language', 'invalid']]));
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
});
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
const s = settings();
s.voiceLanguageSetting.set('fr');
s.voiceProviderSetting.set('whisper');
for (const [file, name, method] of [
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
]) {
const host = { continuousClients: new Map() };
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
// Run each real capture-options method with only its native dependencies stubbed.
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
const member = declaration.members.find(n => n.name?.getText(tree) === method);
const capture = loadBody(member.body.getText(tree), s, host);
assert.equal(capture().language, 'fr');
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
assert.equal(capture().provider, 'soniox');
assert.equal(capture().language, 'ja');
host.continuousClients.clear();
assert.equal(capture().language, 'fr');
}
});
function loadBody(body, s, host) {
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
}
test('multilingual downloads and deletion leave an existing English model intact', () => {
const files = new Map(), downloads = [];
const asr = load('app/native/asr-model.ts', () => ({
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
}), {
java: { io: { File: class {
constructor(name) { this.name = name; }
exists() { return files.has(this.name); }
length() { return files.get(this.name) ?? 0; }
delete() { return files.delete(this.name); }
} } },
com: { faceclaw: { app: {
FaceclawModelDownloaderListener: function(listener) { return listener; },
FaceclawModelDownloader: class {
constructor(url, destination, hash, size, listener) {
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
}
start() {}
cancel() {}
},
} } },
});
const english = asr.ASR_MODELS['whisper-base-en'];
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
assert.equal(asr.isAsrModelReady('whisper-base'), false);
asr.startAsrModelDownload('whisper-base');
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
for (let index = 0; index < 3; index++) {
const download = downloads[index];
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
assert.match(download.hash, /^[a-f0-9]{64}$/);
files.set(download.destination, download.size);
download.listener.onDone();
}
assert.equal(downloads.length, 3);
assert.equal(asr.isAsrModelReady('whisper-base'), true);
asr.deleteAsrModel('whisper-base');
assert.equal(asr.isAsrModelReady('whisper-base'), false);
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
});