forked from Blendman974/faceclaw
Add initial voice-command support
Adds a single wakeword-style voice command, "screen on", and deals with all the wiring and permissions that entails. Currently this only supports using the phone's mic (or a bluetooth headset), not the G2 mic.
This commit is contained in:
1 parent
c4cb4f9f01
commit
9fbb985db6
35 files changed
+1722
-16
No files matched your search
@@ -27,7 +27,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
|
||||
private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE;
|
||||
private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT;
|
||||
private static final int DESIRED_MTU = 247;
|
||||
private static final int DESIRED_MTU = 512;
|
||||
private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH;
|
||||
|
||||
private static final int CONNECT_TIMEOUT_MS = 5_000;
|
||||
@@ -40,7 +40,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000;
|
||||
private static final int BATTERY_INPUT_QUIET_MS = 5_000;
|
||||
private static final int WINDOW_SIZE = 3;
|
||||
private static final int IMAGE_FRAGMENT_SIZE = 1000;
|
||||
private static final int IMAGE_FRAGMENT_SIZE = 3800;
|
||||
private static final int IMAGE_RETRY_DELAY_MS = 10_000;
|
||||
private static final boolean IMAGE_FRAGMENT_NO_ACK = false;
|
||||
private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000;
|
||||
|
||||
+11
-1
@@ -6,6 +6,7 @@ import android.app.NotificationManager;
|
||||
import android.app.PendingIntent;
|
||||
import android.app.Service;
|
||||
import android.content.Intent;
|
||||
import android.content.pm.ServiceInfo;
|
||||
import android.os.Build;
|
||||
import android.os.IBinder;
|
||||
|
||||
@@ -41,7 +42,11 @@ public class FaceclawForegroundService extends Service {
|
||||
text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected"
|
||||
);
|
||||
|
||||
startForeground(NOTIFICATION_ID, notification);
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIFICATION_ID, notification, foregroundServiceType());
|
||||
} else {
|
||||
startForeground(NOTIFICATION_ID, notification);
|
||||
}
|
||||
|
||||
if (ACTION_UPDATE.equals(action)) {
|
||||
NotificationManager manager = getSystemService(NotificationManager.class);
|
||||
@@ -95,4 +100,9 @@ public class FaceclawForegroundService extends Service {
|
||||
.setOnlyAlertOnce(true)
|
||||
.build();
|
||||
}
|
||||
|
||||
private int foregroundServiceType() {
|
||||
return ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE
|
||||
| ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.Manifest;
|
||||
import android.bluetooth.BluetoothAdapter;
|
||||
import android.bluetooth.BluetoothDevice;
|
||||
import android.bluetooth.BluetoothManager;
|
||||
import android.bluetooth.BluetoothProfile;
|
||||
import android.content.Context;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.res.AssetManager;
|
||||
import android.media.AudioDeviceInfo;
|
||||
import android.media.AudioFormat;
|
||||
import android.media.AudioManager;
|
||||
import android.media.AudioRecord;
|
||||
import android.media.MediaRecorder;
|
||||
import android.os.Build;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.core.content.ContextCompat;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotter;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
|
||||
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Set;
|
||||
|
||||
public class FaceclawVoiceController {
|
||||
private static final String TAG = "FaceclawVoice";
|
||||
private static final int SAMPLE_RATE = 16000;
|
||||
private static final int FEATURE_DIM = 80;
|
||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||
private static final String[] MODEL_FILES = {
|
||||
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"tokens.txt",
|
||||
"screen-on-keywords.txt"
|
||||
};
|
||||
|
||||
private final Context appContext;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
private volatile FaceclawVoiceControllerListener listener;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
private AudioRecord audioRecord;
|
||||
private KeywordSpotter keywordSpotter;
|
||||
private OnlineStream stream;
|
||||
|
||||
public FaceclawVoiceController(Context context) {
|
||||
this.appContext = context.getApplicationContext();
|
||||
}
|
||||
|
||||
public void setListener(FaceclawVoiceControllerListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
return;
|
||||
}
|
||||
if (!hasRecordAudioPermission()) {
|
||||
emitStatus("Voice control needs microphone permission.");
|
||||
return;
|
||||
}
|
||||
if (!hasBluetoothHeadset()) {
|
||||
emitStatus("Voice control needs a paired Bluetooth headset.");
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return;
|
||||
}
|
||||
started = false;
|
||||
threadToJoin = workerThread;
|
||||
}
|
||||
stopAudio();
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
stop();
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
emitStatus("Voice control loading wake-word model...");
|
||||
File modelDir = installModelFiles();
|
||||
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
||||
stream = keywordSpotter.createStream();
|
||||
if (!startAudio()) {
|
||||
emitStatus("Voice control could not start microphone input.");
|
||||
return;
|
||||
}
|
||||
|
||||
emitStatus("Voice control listening for \"screen on\".");
|
||||
processAudio();
|
||||
} catch (Throwable error) {
|
||||
Log.e(TAG, "Voice control failed", error);
|
||||
emitStatus("Voice control failed: " + error.getMessage());
|
||||
} finally {
|
||||
releaseSherpa();
|
||||
stopAudio();
|
||||
synchronized (lock) {
|
||||
started = false;
|
||||
workerThread = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private KeywordSpotterConfig buildConfig(File modelDir) {
|
||||
return KeywordSpotterConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setOnlineModelConfig(OnlineModelConfig.builder()
|
||||
.setTransducer(OnlineTransducerModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setModelType("zipformer2")
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
|
||||
.setKeywordsScore(1.5f)
|
||||
.setKeywordsThreshold(0.35f)
|
||||
.build();
|
||||
}
|
||||
|
||||
private File installModelFiles() throws IOException {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
|
||||
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||
throw new IOException("Could not create " + modelDir.getAbsolutePath());
|
||||
}
|
||||
AssetManager assets = appContext.getAssets();
|
||||
for (String fileName : MODEL_FILES) {
|
||||
copyAssetIfNeeded(
|
||||
assets,
|
||||
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
|
||||
new File(modelDir, fileName)
|
||||
);
|
||||
}
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
|
||||
if (destination.exists() && destination.length() > 0) {
|
||||
return;
|
||||
}
|
||||
try (InputStream input = assets.open(assetPath);
|
||||
FileOutputStream output = new FileOutputStream(destination)) {
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int read;
|
||||
while ((read = input.read(buffer)) >= 0) {
|
||||
output.write(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean startAudio() {
|
||||
routeToBluetoothHeadset();
|
||||
int minBufferBytes = AudioRecord.getMinBufferSize(
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
);
|
||||
if (minBufferBytes <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (!hasRecordAudioPermission()) {
|
||||
return false;
|
||||
}
|
||||
audioRecord = new AudioRecord(
|
||||
MediaRecorder.AudioSource.VOICE_RECOGNITION,
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT,
|
||||
minBufferBytes * 2
|
||||
);
|
||||
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
|
||||
audioRecord.release();
|
||||
audioRecord = null;
|
||||
return false;
|
||||
}
|
||||
audioRecord.startRecording();
|
||||
return true;
|
||||
}
|
||||
|
||||
private void processAudio() {
|
||||
short[] buffer = new short[SAMPLE_RATE / 10];
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
AudioRecord currentAudio = audioRecord;
|
||||
OnlineStream currentStream = stream;
|
||||
KeywordSpotter currentSpotter = keywordSpotter;
|
||||
if (currentAudio == null || currentStream == null || currentSpotter == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int count = currentAudio.read(buffer, 0, buffer.length);
|
||||
if (count <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = buffer[i] / 32768.0f;
|
||||
}
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
|
||||
while (currentSpotter.isReady(currentStream)) {
|
||||
currentSpotter.decode(currentStream);
|
||||
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
|
||||
String keyword = result == null ? "" : result.getKeyword();
|
||||
if (keyword != null && keyword.trim().length() > 0) {
|
||||
currentSpotter.reset(currentStream);
|
||||
emitWakeWord(keyword);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void stopAudio() {
|
||||
AudioRecord record = audioRecord;
|
||||
audioRecord = null;
|
||||
if (record != null) {
|
||||
try {
|
||||
record.stop();
|
||||
} catch (IllegalStateException ignored) {
|
||||
}
|
||||
record.release();
|
||||
}
|
||||
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
|
||||
if (audioManager != null) {
|
||||
if (Build.VERSION.SDK_INT >= 31) {
|
||||
audioManager.clearCommunicationDevice();
|
||||
}
|
||||
audioManager.stopBluetoothSco();
|
||||
audioManager.setBluetoothScoOn(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void releaseSherpa() {
|
||||
if (stream != null) {
|
||||
stream.release();
|
||||
stream = null;
|
||||
}
|
||||
if (keywordSpotter != null) {
|
||||
keywordSpotter.release();
|
||||
keywordSpotter = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void routeToBluetoothHeadset() {
|
||||
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
|
||||
if (audioManager == null) {
|
||||
return;
|
||||
}
|
||||
audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
|
||||
audioManager.startBluetoothSco();
|
||||
audioManager.setBluetoothScoOn(true);
|
||||
if (Build.VERSION.SDK_INT >= 31) {
|
||||
for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
|
||||
if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
|
||||
|| device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
|
||||
audioManager.setCommunicationDevice(device);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasBluetoothHeadset() {
|
||||
if (Build.VERSION.SDK_INT >= 31
|
||||
&& ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
|
||||
!= PackageManager.PERMISSION_GRANTED) {
|
||||
return false;
|
||||
}
|
||||
BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
|
||||
BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
|
||||
if (adapter == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
|
||||
if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
|
||||
return true;
|
||||
}
|
||||
Set<BluetoothDevice> bondedDevices = adapter.getBondedDevices();
|
||||
return bondedDevices != null && !bondedDevices.isEmpty();
|
||||
} catch (SecurityException ignored) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasRecordAudioPermission() {
|
||||
return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
||||
== PackageManager.PERMISSION_GRANTED;
|
||||
}
|
||||
|
||||
private void emitStatus(String status) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onStatus(status));
|
||||
}
|
||||
|
||||
private void emitWakeWord(String keyword) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onWakeWord(keyword));
|
||||
}
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawVoiceControllerListener {
|
||||
void onStatus(String status);
|
||||
|
||||
void onWakeWord(String keyword);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class FeatureConfig {
|
||||
private final int sampleRate;
|
||||
private final int featureDim;
|
||||
private final float dither;
|
||||
|
||||
private FeatureConfig(Builder builder) {
|
||||
this.sampleRate = builder.sampleRate;
|
||||
this.featureDim = builder.featureDim;
|
||||
this.dither = builder.dither;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return sampleRate;
|
||||
}
|
||||
|
||||
public int getFeatureDim() {
|
||||
return featureDim;
|
||||
}
|
||||
|
||||
public float getDither() {
|
||||
return dither;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private int sampleRate = 16000;
|
||||
private int featureDim = 80;
|
||||
private float dither = 0.0f;
|
||||
|
||||
public Builder setSampleRate(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setFeatureDim(int featureDim) {
|
||||
this.featureDim = featureDim;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDither(float dither) {
|
||||
this.dither = dither;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FeatureConfig build() {
|
||||
return new FeatureConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotter {
|
||||
private long ptr;
|
||||
|
||||
public KeywordSpotter(KeywordSpotterConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
|
||||
}
|
||||
}
|
||||
|
||||
public OnlineStream createStream() {
|
||||
return createStream("");
|
||||
}
|
||||
|
||||
public OnlineStream createStream(String keywords) {
|
||||
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
|
||||
}
|
||||
|
||||
public boolean isReady(OnlineStream stream) {
|
||||
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void decode(OnlineStream stream) {
|
||||
decode(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void reset(OnlineStream stream) {
|
||||
reset(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public KeywordSpotterResult getResult(OnlineStream stream) {
|
||||
return getResult(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(ptr);
|
||||
ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native long newFromFile(KeywordSpotterConfig config);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long createStream(long ptr, String keywords);
|
||||
|
||||
private native boolean isReady(long ptr, long streamPtr);
|
||||
|
||||
private native void decode(long ptr, long streamPtr);
|
||||
|
||||
private native void reset(long ptr, long streamPtr);
|
||||
|
||||
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotterConfig {
|
||||
private final FeatureConfig featConfig;
|
||||
private final OnlineModelConfig modelConfig;
|
||||
private final int maxActivePaths;
|
||||
private final String keywordsFile;
|
||||
private final float keywordsScore;
|
||||
private final float keywordsThreshold;
|
||||
private final int numTrailingBlanks;
|
||||
|
||||
private KeywordSpotterConfig(Builder builder) {
|
||||
this.featConfig = builder.featConfig;
|
||||
this.modelConfig = builder.modelConfig;
|
||||
this.maxActivePaths = builder.maxActivePaths;
|
||||
this.keywordsFile = builder.keywordsFile;
|
||||
this.keywordsScore = builder.keywordsScore;
|
||||
this.keywordsThreshold = builder.keywordsThreshold;
|
||||
this.numTrailingBlanks = builder.numTrailingBlanks;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public FeatureConfig getFeatConfig() {
|
||||
return featConfig;
|
||||
}
|
||||
|
||||
public OnlineModelConfig getModelConfig() {
|
||||
return modelConfig;
|
||||
}
|
||||
|
||||
public int getMaxActivePaths() {
|
||||
return maxActivePaths;
|
||||
}
|
||||
|
||||
public String getKeywordsFile() {
|
||||
return keywordsFile;
|
||||
}
|
||||
|
||||
public float getKeywordsScore() {
|
||||
return keywordsScore;
|
||||
}
|
||||
|
||||
public float getKeywordsThreshold() {
|
||||
return keywordsThreshold;
|
||||
}
|
||||
|
||||
public int getNumTrailingBlanks() {
|
||||
return numTrailingBlanks;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private FeatureConfig featConfig = FeatureConfig.builder().build();
|
||||
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
|
||||
private int maxActivePaths = 4;
|
||||
private String keywordsFile = "keywords.txt";
|
||||
private float keywordsScore = 1.5f;
|
||||
private float keywordsThreshold = 0.25f;
|
||||
private int numTrailingBlanks = 2;
|
||||
|
||||
public Builder setFeatureConfig(FeatureConfig featConfig) {
|
||||
this.featConfig = featConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
|
||||
this.modelConfig = modelConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsFile(String keywordsFile) {
|
||||
this.keywordsFile = keywordsFile;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsScore(float keywordsScore) {
|
||||
this.keywordsScore = keywordsScore;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsThreshold(float keywordsThreshold) {
|
||||
this.keywordsThreshold = keywordsThreshold;
|
||||
return this;
|
||||
}
|
||||
|
||||
public KeywordSpotterConfig build() {
|
||||
return new KeywordSpotterConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotterResult {
|
||||
private final String keyword;
|
||||
private final String[] tokens;
|
||||
private final float[] timestamps;
|
||||
|
||||
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
|
||||
this.keyword = keyword == null ? "" : keyword;
|
||||
this.tokens = tokens == null ? new String[0] : tokens;
|
||||
this.timestamps = timestamps == null ? new float[0] : timestamps;
|
||||
}
|
||||
|
||||
public String getKeyword() {
|
||||
return keyword;
|
||||
}
|
||||
|
||||
public String[] getTokens() {
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public float[] getTimestamps() {
|
||||
return timestamps;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineModelConfig {
|
||||
private final OnlineTransducerModelConfig transducer;
|
||||
private final OnlineParaformerModelConfig paraformer;
|
||||
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
|
||||
private final OnlineNeMoCtcModelConfig neMoCtc;
|
||||
private final OnlineToneCtcModelConfig toneCtc;
|
||||
private final String tokens;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
private final String modelType;
|
||||
private final String modelingUnit;
|
||||
private final String bpeVocab;
|
||||
|
||||
private OnlineModelConfig(Builder builder) {
|
||||
this.transducer = builder.transducer;
|
||||
this.paraformer = builder.paraformer;
|
||||
this.zipformer2Ctc = builder.zipformer2Ctc;
|
||||
this.neMoCtc = builder.neMoCtc;
|
||||
this.toneCtc = builder.toneCtc;
|
||||
this.tokens = builder.tokens;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
this.modelType = builder.modelType;
|
||||
this.modelingUnit = builder.modelingUnit;
|
||||
this.bpeVocab = builder.bpeVocab;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public OnlineTransducerModelConfig getTransducer() {
|
||||
return transducer;
|
||||
}
|
||||
|
||||
public OnlineParaformerModelConfig getParaformer() {
|
||||
return paraformer;
|
||||
}
|
||||
|
||||
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
|
||||
return zipformer2Ctc;
|
||||
}
|
||||
|
||||
public OnlineNeMoCtcModelConfig getNeMoCtc() {
|
||||
return neMoCtc;
|
||||
}
|
||||
|
||||
public OnlineToneCtcModelConfig getToneCtc() {
|
||||
return toneCtc;
|
||||
}
|
||||
|
||||
public String getTokens() {
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public int getNumThreads() {
|
||||
return numThreads;
|
||||
}
|
||||
|
||||
public boolean getDebug() {
|
||||
return debug;
|
||||
}
|
||||
|
||||
public String getProvider() {
|
||||
return provider;
|
||||
}
|
||||
|
||||
public String getModelType() {
|
||||
return modelType;
|
||||
}
|
||||
|
||||
public String getModelingUnit() {
|
||||
return modelingUnit;
|
||||
}
|
||||
|
||||
public String getBpeVocab() {
|
||||
return bpeVocab;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
|
||||
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
|
||||
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
|
||||
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
|
||||
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
|
||||
private String tokens = "";
|
||||
private int numThreads = 1;
|
||||
private boolean debug = false;
|
||||
private String provider = "cpu";
|
||||
private String modelType = "";
|
||||
private String modelingUnit = "cjkchar";
|
||||
private String bpeVocab = "";
|
||||
|
||||
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
|
||||
this.transducer = transducer;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTokens(String tokens) {
|
||||
this.tokens = tokens;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setModelType(String modelType) {
|
||||
this.modelType = modelType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setModelingUnit(String modelingUnit) {
|
||||
this.modelingUnit = modelingUnit;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setBpeVocab(String bpeVocab) {
|
||||
this.bpeVocab = bpeVocab;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineModelConfig build() {
|
||||
return new OnlineModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineNeMoCtcModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineNeMoCtcModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineNeMoCtcModelConfig build() {
|
||||
return new OnlineNeMoCtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineParaformerModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
|
||||
private OnlineParaformerModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineParaformerModelConfig build() {
|
||||
return new OnlineParaformerModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineStream {
|
||||
private long ptr;
|
||||
|
||||
public OnlineStream(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public long getPtr() {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
public void acceptWaveform(float[] samples, int sampleRate) {
|
||||
acceptWaveform(ptr, samples, sampleRate);
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(ptr);
|
||||
ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
|
||||
|
||||
private native void delete(long ptr);
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineToneCtcModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineToneCtcModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineToneCtcModelConfig build() {
|
||||
return new OnlineToneCtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineTransducerModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
private final String joiner;
|
||||
|
||||
private OnlineTransducerModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
this.joiner = builder.joiner;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public String getJoiner() {
|
||||
return joiner;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
private String joiner = "";
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setJoiner(String joiner) {
|
||||
this.joiner = joiner;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineTransducerModelConfig build() {
|
||||
return new OnlineTransducerModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineZipformer2CtcModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineZipformer2CtcModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineZipformer2CtcModelConfig build() {
|
||||
return new OnlineZipformer2CtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user