From f946c11cea4ab4fbf8eb69dd6d9d85c04a1ddb06 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 7 May 2026 17:47:30 -0700 Subject: [PATCH] WIP on-device voice model --- App_Resources/Android/app.gradle | 14 +- .../com/k2fsa/sherpa/onnx/EndpointConfig.java | 66 ---- .../com/k2fsa/sherpa/onnx/EndpointRule.java | 54 --- .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 325 ++++++++++++++++++ .../onnx/OfflineMoonshineModelConfig.java | 78 +++++ .../k2fsa/sherpa/onnx/OfflineRecognizer.java | 44 +++ ...nfig.java => OfflineRecognizerConfig.java} | 54 +-- .../sherpa/onnx/OfflineRecognizerResult.java | 57 +++ .../com/k2fsa/sherpa/onnx/OfflineStream.java | 29 ++ .../onnx/OnlineCtcFstDecoderConfig.java | 42 --- .../com/k2fsa/sherpa/onnx/OnlineLMConfig.java | 42 --- .../k2fsa/sherpa/onnx/OnlineRecognizer.java | 66 ---- .../sherpa/onnx/OnlineRecognizerResult.java | 31 -- .../java/com/k2fsa/sherpa/onnx/QnnConfig.java | 13 + 14 files changed, 560 insertions(+), 355 deletions(-) delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.java rename App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/{OnlineRecognizerConfig.java => OfflineRecognizerConfig.java} (59%) create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.java delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java delete mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.java diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 457e23c4..23cac499 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -5,9 +5,9 @@ dependencies { // implementation 'androidx.multidex:multidex:2.0.1' } -def faceclawSherpaOnnxVersion = "1.12.34" +def faceclawSherpaOnnxVersion = "1.13.0" def faceclawLibLc3Version = "1.1.3" -def faceclawAsrModelName = "sherpa-onnx-streaming-zipformer-en-2023-06-26" +def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" def faceclawAbi = "arm64-v8a" def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi") def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr") @@ -59,9 +59,8 @@ tasks.register("prepareFaceclawNativeLibs") { file("$faceclawJniLibsDir/libonnxruntime.so"), file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"), file("$faceclawJniLibsDir/libfaceclaw_lc3.so"), - file("$faceclawAssetsDir/$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx"), - file("$faceclawAssetsDir/$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx"), - file("$faceclawAssetsDir/$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx"), + file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"), + file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"), file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt") ) @@ -91,9 +90,8 @@ tasks.register("prepareFaceclawNativeLibs") { ) copy { from tarTree(resources.bzip2(asrArchive)) - include "$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx" - include "$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx" - include "$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx" + include "$faceclawAsrModelName/encoder_model.ort" + include "$faceclawAsrModelName/decoder_model_merged.ort" include "$faceclawAsrModelName/tokens.txt" includeEmptyDirs = false into faceclawAssetsDir diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java deleted file mode 100644 index 76a768a8..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class EndpointConfig { - private final EndpointRule rule1; - private final EndpointRule rule2; - private final EndpointRule rule3; - - private EndpointConfig(Builder builder) { - this.rule1 = builder.rule1; - this.rule2 = builder.rule2; - this.rule3 = builder.rule3; - } - - public static Builder builder() { - return new Builder(); - } - - public EndpointRule getRule1() { - return rule1; - } - - public EndpointRule getRule2() { - return rule2; - } - - public EndpointRule getRule3() { - return rule3; - } - - public static class Builder { - private EndpointRule rule1 = EndpointRule.builder() - .setMustContainNonSilence(false) - .setMinTrailingSilence(2.4f) - .setMinUtteranceLength(0.0f) - .build(); - private EndpointRule rule2 = EndpointRule.builder() - .setMustContainNonSilence(true) - .setMinTrailingSilence(1.2f) - .setMinUtteranceLength(0.0f) - .build(); - private EndpointRule rule3 = EndpointRule.builder() - .setMustContainNonSilence(false) - .setMinTrailingSilence(0.0f) - .setMinUtteranceLength(20.0f) - .build(); - - public Builder setRule1(EndpointRule rule1) { - this.rule1 = rule1; - return this; - } - - public Builder setRule2(EndpointRule rule2) { - this.rule2 = rule2; - return this; - } - - public Builder setRule3(EndpointRule rule3) { - this.rule3 = rule3; - return this; - } - - public EndpointConfig build() { - return new EndpointConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java deleted file mode 100644 index b71bdf0b..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class EndpointRule { - private final boolean mustContainNonSilence; - private final float minTrailingSilence; - private final float minUtteranceLength; - - private EndpointRule(Builder builder) { - this.mustContainNonSilence = builder.mustContainNonSilence; - this.minTrailingSilence = builder.minTrailingSilence; - this.minUtteranceLength = builder.minUtteranceLength; - } - - public static Builder builder() { - return new Builder(); - } - - public boolean getMustContainNonSilence() { - return mustContainNonSilence; - } - - public float getMinTrailingSilence() { - return minTrailingSilence; - } - - public float getMinUtteranceLength() { - return minUtteranceLength; - } - - public static class Builder { - private boolean mustContainNonSilence = false; - private float minTrailingSilence = 0.0f; - private float minUtteranceLength = 0.0f; - - public Builder setMustContainNonSilence(boolean mustContainNonSilence) { - this.mustContainNonSilence = mustContainNonSilence; - return this; - } - - public Builder setMinTrailingSilence(float minTrailingSilence) { - this.minTrailingSilence = minTrailingSilence; - return this; - } - - public Builder setMinUtteranceLength(float minUtteranceLength) { - this.minUtteranceLength = minUtteranceLength; - return this; - } - - public EndpointRule build() { - return new EndpointRule(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java new file mode 100644 index 00000000..0d916bf8 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -0,0 +1,325 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineModelConfig { + private final OfflineTransducerModelConfig transducer; + private final OfflineParaformerModelConfig paraformer; + private final OfflineWhisperModelConfig whisper; + private final OfflineFireRedAsrModelConfig fireRedAsr; + private final OfflineMoonshineModelConfig moonshine; + private final OfflineNemoEncDecCtcModelConfig nemo; + private final OfflineSenseVoiceModelConfig senseVoice; + private final OfflineDolphinModelConfig dolphin; + private final OfflineZipformerCtcModelConfig zipformerCtc; + private final OfflineWenetCtcModelConfig wenetCtc; + private final OfflineOmnilingualAsrCtcModelConfig omnilingual; + private final OfflineMedAsrCtcModelConfig medasr; + private final OfflineFunAsrNanoModelConfig funasrNano; + private final OfflineQwen3AsrModelConfig qwen3Asr; + private final OfflineFireRedAsrCtcModelConfig fireRedAsrCtc; + private final OfflineCanaryModelConfig canary; + private final OfflineCohereTranscribeModelConfig cohereTranscribe; + private final String teleSpeech; + private final int numThreads; + private final boolean debug; + private final String provider; + private final String modelType; + private final String tokens; + private final String modelingUnit; + private final String bpeVocab; + + private OfflineModelConfig(Builder builder) { + this.transducer = builder.transducer; + this.paraformer = builder.paraformer; + this.whisper = builder.whisper; + this.fireRedAsr = builder.fireRedAsr; + this.moonshine = builder.moonshine; + this.nemo = builder.nemo; + this.senseVoice = builder.senseVoice; + this.dolphin = builder.dolphin; + this.zipformerCtc = builder.zipformerCtc; + this.wenetCtc = builder.wenetCtc; + this.omnilingual = builder.omnilingual; + this.medasr = builder.medasr; + this.funasrNano = builder.funasrNano; + this.qwen3Asr = builder.qwen3Asr; + this.fireRedAsrCtc = builder.fireRedAsrCtc; + this.canary = builder.canary; + this.cohereTranscribe = builder.cohereTranscribe; + this.teleSpeech = builder.teleSpeech; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + this.modelType = builder.modelType; + this.tokens = builder.tokens; + this.modelingUnit = builder.modelingUnit; + this.bpeVocab = builder.bpeVocab; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineTransducerModelConfig getTransducer() { + return transducer; + } + + public OfflineParaformerModelConfig getParaformer() { + return paraformer; + } + + public OfflineWhisperModelConfig getWhisper() { + return whisper; + } + + public OfflineFireRedAsrModelConfig getFireRedAsr() { + return fireRedAsr; + } + + public OfflineMoonshineModelConfig getMoonshine() { + return moonshine; + } + + public OfflineNemoEncDecCtcModelConfig getNemo() { + return nemo; + } + + public OfflineSenseVoiceModelConfig getSenseVoice() { + return senseVoice; + } + + public OfflineDolphinModelConfig getDolphin() { + return dolphin; + } + + public OfflineZipformerCtcModelConfig getZipformerCtc() { + return zipformerCtc; + } + + public OfflineWenetCtcModelConfig getWenetCtc() { + return wenetCtc; + } + + public OfflineOmnilingualAsrCtcModelConfig getOmnilingual() { + return omnilingual; + } + + public OfflineMedAsrCtcModelConfig getMedasr() { + return medasr; + } + + public OfflineFunAsrNanoModelConfig getFunasrNano() { + return funasrNano; + } + + public OfflineQwen3AsrModelConfig getQwen3Asr() { + return qwen3Asr; + } + + public OfflineFireRedAsrCtcModelConfig getFireRedAsrCtc() { + return fireRedAsrCtc; + } + + public OfflineCanaryModelConfig getCanary() { + return canary; + } + + public OfflineCohereTranscribeModelConfig getCohereTranscribe() { + return cohereTranscribe; + } + + public String getTeleSpeech() { + return teleSpeech; + } + + public int getNumThreads() { + return numThreads; + } + + public boolean getDebug() { + return debug; + } + + public String getProvider() { + return provider; + } + + public String getModelType() { + return modelType; + } + + public String getTokens() { + return tokens; + } + + public String getModelingUnit() { + return modelingUnit; + } + + public String getBpeVocab() { + return bpeVocab; + } + + public static class Builder { + private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig(); + private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig(); + private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig(); + private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig(); + private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build(); + private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig(); + private OfflineSenseVoiceModelConfig senseVoice = new OfflineSenseVoiceModelConfig(); + private OfflineDolphinModelConfig dolphin = new OfflineDolphinModelConfig(); + private OfflineZipformerCtcModelConfig zipformerCtc = new OfflineZipformerCtcModelConfig(); + private OfflineWenetCtcModelConfig wenetCtc = new OfflineWenetCtcModelConfig(); + private OfflineOmnilingualAsrCtcModelConfig omnilingual = new OfflineOmnilingualAsrCtcModelConfig(); + private OfflineMedAsrCtcModelConfig medasr = new OfflineMedAsrCtcModelConfig(); + private OfflineFunAsrNanoModelConfig funasrNano = new OfflineFunAsrNanoModelConfig(); + private OfflineQwen3AsrModelConfig qwen3Asr = new OfflineQwen3AsrModelConfig(); + private OfflineFireRedAsrCtcModelConfig fireRedAsrCtc = new OfflineFireRedAsrCtcModelConfig(); + private OfflineCanaryModelConfig canary = new OfflineCanaryModelConfig(); + private OfflineCohereTranscribeModelConfig cohereTranscribe = new OfflineCohereTranscribeModelConfig(); + private String teleSpeech = ""; + private int numThreads = 1; + private boolean debug = false; + private String provider = "cpu"; + private String modelType = ""; + private String tokens = ""; + private String modelingUnit = ""; + private String bpeVocab = ""; + + public Builder setMoonshine(OfflineMoonshineModelConfig moonshine) { + this.moonshine = moonshine; + return this; + } + + public Builder setTokens(String tokens) { + this.tokens = tokens; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + + public Builder setModelType(String modelType) { + this.modelType = modelType; + return this; + } + + public OfflineModelConfig build() { + return new OfflineModelConfig(this); + } + } +} + +class OfflineTransducerModelConfig { + private final String encoder = ""; + private final String decoder = ""; + private final String joiner = ""; +} + +class OfflineParaformerModelConfig { + private final String model = ""; + private final QnnConfig qnnConfig = new QnnConfig(); +} + +class OfflineWhisperModelConfig { + private final String encoder = ""; + private final String decoder = ""; + private final String language = "en"; + private final String task = "transcribe"; + private final int tailPaddings = 1000; + private final boolean enableTokenTimestamps = false; + private final boolean enableSegmentTimestamps = false; +} + +class OfflineFireRedAsrModelConfig { + private final String encoder = ""; + private final String decoder = ""; +} + +class OfflineNemoEncDecCtcModelConfig { + private final String model = ""; +} + +class OfflineSenseVoiceModelConfig { + private final String model = ""; + private final String language = ""; + private final boolean useInverseTextNormalization = true; + private final QnnConfig qnnConfig = new QnnConfig(); +} + +class OfflineDolphinModelConfig { + private final String model = ""; +} + +class OfflineZipformerCtcModelConfig { + private final String model = ""; + private final QnnConfig qnnConfig = new QnnConfig(); +} + +class OfflineWenetCtcModelConfig { + private final String model = ""; +} + +class OfflineOmnilingualAsrCtcModelConfig { + private final String model = ""; +} + +class OfflineMedAsrCtcModelConfig { + private final String model = ""; +} + +class OfflineFunAsrNanoModelConfig { + private final String encoderAdaptor = ""; + private final String llm = ""; + private final String embedding = ""; + private final String tokenizer = ""; + private final String systemPrompt = "You are a helpful assistant."; + private final String userPrompt = "Speech transcription:"; + private final int maxNewTokens = 512; + private final float temperature = 1e-6f; + private final float topP = 0.8f; + private final int seed = 42; + private final String language = ""; + private final boolean itn = true; + private final String hotwords = ""; +} + +class OfflineQwen3AsrModelConfig { + private final String convFrontend = ""; + private final String encoder = ""; + private final String decoder = ""; + private final String tokenizer = ""; + private final int maxTotalLen = 512; + private final int maxNewTokens = 128; + private final float temperature = 1e-6f; + private final float topP = 0.8f; + private final int seed = 42; + private final String hotwords = ""; +} + +class OfflineFireRedAsrCtcModelConfig { + private final String model = ""; +} + +class OfflineCanaryModelConfig { + private final String encoder = ""; + private final String decoder = ""; + private final String srcLang = "en"; + private final String tgtLang = "en"; + private final boolean usePnc = true; +} + +class OfflineCohereTranscribeModelConfig { + private final String encoder = ""; + private final String decoder = ""; + private final String language = ""; + private final boolean usePunct = true; + private final boolean useItn = true; +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java new file mode 100644 index 00000000..7be13a09 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java @@ -0,0 +1,78 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineMoonshineModelConfig { + private final String preprocessor; + private final String encoder; + private final String uncachedDecoder; + private final String cachedDecoder; + private final String mergedDecoder; + + private OfflineMoonshineModelConfig(Builder builder) { + this.preprocessor = builder.preprocessor; + this.encoder = builder.encoder; + this.uncachedDecoder = builder.uncachedDecoder; + this.cachedDecoder = builder.cachedDecoder; + this.mergedDecoder = builder.mergedDecoder; + } + + public static Builder builder() { + return new Builder(); + } + + public String getPreprocessor() { + return preprocessor; + } + + public String getEncoder() { + return encoder; + } + + public String getUncachedDecoder() { + return uncachedDecoder; + } + + public String getCachedDecoder() { + return cachedDecoder; + } + + public String getMergedDecoder() { + return mergedDecoder; + } + + public static class Builder { + private String preprocessor = ""; + private String encoder = ""; + private String uncachedDecoder = ""; + private String cachedDecoder = ""; + private String mergedDecoder = ""; + + public Builder setPreprocessor(String preprocessor) { + this.preprocessor = preprocessor; + return this; + } + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setUncachedDecoder(String uncachedDecoder) { + this.uncachedDecoder = uncachedDecoder; + return this; + } + + public Builder setCachedDecoder(String cachedDecoder) { + this.cachedDecoder = cachedDecoder; + return this; + } + + public Builder setMergedDecoder(String mergedDecoder) { + this.mergedDecoder = mergedDecoder; + return this; + } + + public OfflineMoonshineModelConfig build() { + return new OfflineMoonshineModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.java new file mode 100644 index 00000000..83780841 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.java @@ -0,0 +1,44 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineRecognizer { + private long ptr; + + public OfflineRecognizer(OfflineRecognizerConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid OfflineRecognizerConfig"); + } + } + + public OfflineStream createStream() { + return new OfflineStream(createStream(ptr)); + } + + public void decode(OfflineStream stream) { + if (stream != null) { + decode(ptr, stream.getPtr()); + } + } + + public OfflineRecognizerResult getResult(OfflineStream stream) { + if (stream == null) { + return new OfflineRecognizerResult("", new String[0], new float[0], "", "", "", new float[0]); + } + return getResult(stream.getPtr()); + } + + public void release() { + if (ptr != 0) { + delete(ptr); + ptr = 0; + } + } + + private native long newFromFile(OfflineRecognizerConfig config); + private native void delete(long ptr); + private native long createStream(long ptr); + private native void decode(long ptr, long streamPtr); + private native OfflineRecognizerResult getResult(long streamPtr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java similarity index 59% rename from App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java rename to App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java index 234a3f63..da3aff76 100644 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerConfig.java @@ -1,13 +1,9 @@ package com.k2fsa.sherpa.onnx; -public class OnlineRecognizerConfig { +public class OfflineRecognizerConfig { private final FeatureConfig featConfig; - private final OnlineModelConfig modelConfig; - private final OnlineLMConfig lmConfig; - private final OnlineCtcFstDecoderConfig ctcFstDecoderConfig; + private final OfflineModelConfig modelConfig; private final HomophoneReplacerConfig hr; - private final EndpointConfig endpointConfig; - private final boolean enableEndpoint; private final String decodingMethod; private final int maxActivePaths; private final String hotwordsFile; @@ -16,14 +12,10 @@ public class OnlineRecognizerConfig { private final String ruleFars; private final float blankPenalty; - private OnlineRecognizerConfig(Builder builder) { + private OfflineRecognizerConfig(Builder builder) { this.featConfig = builder.featConfig; this.modelConfig = builder.modelConfig; - this.lmConfig = builder.lmConfig; - this.ctcFstDecoderConfig = builder.ctcFstDecoderConfig; this.hr = builder.hr; - this.endpointConfig = builder.endpointConfig; - this.enableEndpoint = builder.enableEndpoint; this.decodingMethod = builder.decodingMethod; this.maxActivePaths = builder.maxActivePaths; this.hotwordsFile = builder.hotwordsFile; @@ -41,30 +33,14 @@ public class OnlineRecognizerConfig { return featConfig; } - public OnlineModelConfig getModelConfig() { + public OfflineModelConfig getModelConfig() { return modelConfig; } - public OnlineLMConfig getLmConfig() { - return lmConfig; - } - - public OnlineCtcFstDecoderConfig getCtcFstDecoderConfig() { - return ctcFstDecoderConfig; - } - public HomophoneReplacerConfig getHr() { return hr; } - public EndpointConfig getEndpointConfig() { - return endpointConfig; - } - - public boolean getEnableEndpoint() { - return enableEndpoint; - } - public String getDecodingMethod() { return decodingMethod; } @@ -95,12 +71,8 @@ public class OnlineRecognizerConfig { public static class Builder { private FeatureConfig featConfig = FeatureConfig.builder().build(); - private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); - private OnlineLMConfig lmConfig = OnlineLMConfig.builder().build(); - private OnlineCtcFstDecoderConfig ctcFstDecoderConfig = OnlineCtcFstDecoderConfig.builder().build(); + private OfflineModelConfig modelConfig = OfflineModelConfig.builder().build(); private HomophoneReplacerConfig hr = HomophoneReplacerConfig.builder().build(); - private EndpointConfig endpointConfig = EndpointConfig.builder().build(); - private boolean enableEndpoint = true; private String decodingMethod = "greedy_search"; private int maxActivePaths = 4; private String hotwordsFile = ""; @@ -114,21 +86,11 @@ public class OnlineRecognizerConfig { return this; } - public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { + public Builder setModelConfig(OfflineModelConfig modelConfig) { this.modelConfig = modelConfig; return this; } - public Builder setEndpointConfig(EndpointConfig endpointConfig) { - this.endpointConfig = endpointConfig; - return this; - } - - public Builder setEnableEndpoint(boolean enableEndpoint) { - this.enableEndpoint = enableEndpoint; - return this; - } - public Builder setDecodingMethod(String decodingMethod) { this.decodingMethod = decodingMethod; return this; @@ -139,8 +101,8 @@ public class OnlineRecognizerConfig { return this; } - public OnlineRecognizerConfig build() { - return new OnlineRecognizerConfig(this); + public OfflineRecognizerConfig build() { + return new OfflineRecognizerConfig(this); } } } diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java new file mode 100644 index 00000000..122d9c3e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizerResult.java @@ -0,0 +1,57 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineRecognizerResult { + private final String text; + private final String[] tokens; + private final float[] timestamps; + private final String lang; + private final String emotion; + private final String event; + private final float[] durations; + + public OfflineRecognizerResult( + String text, + String[] tokens, + float[] timestamps, + String lang, + String emotion, + String event, + float[] durations + ) { + this.text = text == null ? "" : text; + this.tokens = tokens == null ? new String[0] : tokens; + this.timestamps = timestamps == null ? new float[0] : timestamps; + this.lang = lang == null ? "" : lang; + this.emotion = emotion == null ? "" : emotion; + this.event = event == null ? "" : event; + this.durations = durations == null ? new float[0] : durations; + } + + public String getText() { + return text; + } + + public String[] getTokens() { + return tokens; + } + + public float[] getTimestamps() { + return timestamps; + } + + public String getLang() { + return lang; + } + + public String getEmotion() { + return emotion; + } + + public String getEvent() { + return event; + } + + public float[] getDurations() { + return durations; + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.java new file mode 100644 index 00000000..4fba5367 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.java @@ -0,0 +1,29 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineStream { + private long ptr; + + public OfflineStream(long ptr) { + this.ptr = ptr; + } + + public long getPtr() { + return ptr; + } + + public void acceptWaveform(float[] samples, int sampleRate) { + acceptWaveform(ptr, samples, sampleRate); + } + + public void release() { + if (ptr == 0) { + return; + } + delete(ptr); + ptr = 0; + } + + private native void acceptWaveform(long ptr, float[] samples, int sampleRate); + + private native void delete(long ptr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java deleted file mode 100644 index ef8518e6..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineCtcFstDecoderConfig { - private final String graph; - private final int maxActive; - - private OnlineCtcFstDecoderConfig(Builder builder) { - this.graph = builder.graph; - this.maxActive = builder.maxActive; - } - - public static Builder builder() { - return new Builder(); - } - - public String getGraph() { - return graph; - } - - public int getMaxActive() { - return maxActive; - } - - public static class Builder { - private String graph = ""; - private int maxActive = 3000; - - public Builder setGraph(String graph) { - this.graph = graph; - return this; - } - - public Builder setMaxActive(int maxActive) { - this.maxActive = maxActive; - return this; - } - - public OnlineCtcFstDecoderConfig build() { - return new OnlineCtcFstDecoderConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java deleted file mode 100644 index b7dad082..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineLMConfig { - private final String model; - private final float scale; - - private OnlineLMConfig(Builder builder) { - this.model = builder.model; - this.scale = builder.scale; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public float getScale() { - return scale; - } - - public static class Builder { - private String model = ""; - private float scale = 0.5f; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public Builder setScale(float scale) { - this.scale = scale; - return this; - } - - public OnlineLMConfig build() { - return new OnlineLMConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java deleted file mode 100644 index 496a0fe9..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineRecognizer { - private long ptr; - - public OnlineRecognizer(OnlineRecognizerConfig config) { - System.loadLibrary("onnxruntime"); - System.loadLibrary("sherpa-onnx-jni"); - ptr = newFromFile(config); - if (ptr == 0) { - throw new IllegalArgumentException("Invalid OnlineRecognizerConfig"); - } - } - - public OnlineStream createStream() { - return createStream(""); - } - - public OnlineStream createStream(String hotwords) { - long streamPtr = createStream(ptr, hotwords == null ? "" : hotwords); - return new OnlineStream(streamPtr); - } - - public boolean isReady(OnlineStream stream) { - return stream != null && isReady(ptr, stream.getPtr()); - } - - public void decode(OnlineStream stream) { - if (stream != null) { - decode(ptr, stream.getPtr()); - } - } - - public void reset(OnlineStream stream) { - if (stream != null) { - reset(ptr, stream.getPtr()); - } - } - - public boolean isEndpoint(OnlineStream stream) { - return stream != null && isEndpoint(ptr, stream.getPtr()); - } - - public OnlineRecognizerResult getResult(OnlineStream stream) { - if (stream == null) { - return new OnlineRecognizerResult("", new String[0], new float[0], new float[0]); - } - return getResult(ptr, stream.getPtr()); - } - - public void release() { - if (ptr != 0) { - delete(ptr); - ptr = 0; - } - } - - private native long newFromFile(OnlineRecognizerConfig config); - private native void delete(long ptr); - private native long createStream(long ptr, String hotwords); - private native void reset(long ptr, long streamPtr); - private native void decode(long ptr, long streamPtr); - private native boolean isEndpoint(long ptr, long streamPtr); - private native boolean isReady(long ptr, long streamPtr); - private native OnlineRecognizerResult getResult(long ptr, long streamPtr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java deleted file mode 100644 index 5b765675..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineRecognizerResult { - private final String text; - private final String[] tokens; - private final float[] timestamps; - private final float[] ysProbs; - - public OnlineRecognizerResult(String text, String[] tokens, float[] timestamps, float[] ysProbs) { - this.text = text == null ? "" : text; - this.tokens = tokens == null ? new String[0] : tokens; - this.timestamps = timestamps == null ? new float[0] : timestamps; - this.ysProbs = ysProbs == null ? new float[0] : ysProbs; - } - - public String getText() { - return text; - } - - public String[] getTokens() { - return tokens; - } - - public float[] getTimestamps() { - return timestamps; - } - - public float[] getYsProbs() { - return ysProbs; - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.java new file mode 100644 index 00000000..a25baaa0 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.java @@ -0,0 +1,13 @@ +package com.k2fsa.sherpa.onnx; + +public class QnnConfig { + private final String backendLib; + private final String contextBinary; + private final String systemLib; + + public QnnConfig() { + this.backendLib = ""; + this.contextBinary = ""; + this.systemLib = ""; + } +}