feat: select languages for cloud and local Whisper transcription

This commit is contained in:
Blendman974 committed 2026-11-06 04:22:48 +01:00
1 parent 6df52021c8
commit 0b9e4cb367
21 files changed
+375 -39

No files matched your search

@@ -25,18 +25,21 @@ internal object AndroidSpeechEngines {
private const val FEATURE_DIM = 80
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
fun recognizerConfig(
modelDir: File, kind: VoiceModelKind,
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
): OfflineRecognizerConfig {
val modelConfig = OfflineModelConfig.builder()
.setNumThreads(1)
if (kind == VoiceModelKind.WHISPER) {
modelConfig
.setWhisper(OfflineWhisperModelConfig.builder()
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
.setLanguage("en")
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
.setLanguage(whisper.language)
.setTask("transcribe")
.build())
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
.setTokens(File(modelDir, whisper.tokens).absolutePath)
} else {
modelConfig
.setMoonshine(OfflineMoonshineModelConfig.builder()
@@ -32,17 +32,6 @@ class FaceclawVoiceController(context: Context) {
"decoder_model_merged.ort",
"tokens.txt"
)
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
// Whisper re-encodes the whole buffer on every call, so the shared session
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
private val ASR_WHISPER_MODEL_FILES = arrayOf(
"base.en-encoder.int8.onnx",
"base.en-decoder.int8.onnx",
"base.en-tokens.txt"
)
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
// (endpointing, transcript pacing) is tuned for.
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
@@ -84,6 +73,7 @@ class FaceclawVoiceController(context: Context) {
private val mainHandler = Handler(Looper.getMainLooper())
@Volatile
private var communicator: FaceclawBleCommunicator? = null
private var whisperModel = WhisperModelFiles.forLanguage(null)
private val host = object : VoiceCaptureHost {
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
@@ -122,7 +112,7 @@ class FaceclawVoiceController(context: Context) {
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
}
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
@@ -168,6 +158,11 @@ class FaceclawVoiceController(context: Context) {
session.setOnboardModelKind(kind)
}
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
fun setOnboardLanguage(language: String?) {
whisperModel = WhisperModelFiles.forLanguage(language)
}
/**
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
@@ -263,8 +258,8 @@ class FaceclawVoiceController(context: Context) {
*/
private fun findAsrModelDir(kind: VoiceModelKind): File? {
val whisper = kind == VoiceModelKind.WHISPER
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
for (fileName in fileNames) {
val file = File(modelDir, fileName)