forked from Blendman974/faceclaw
Add Transcribe screen
This commit is contained in:
1 parent
811477811e
commit
bcc30545ad
16 files changed
+904
-20
No files matched your search
@@ -20,6 +20,8 @@ yarn-error.log*
|
|||||||
tmp/
|
tmp/
|
||||||
typings/
|
typings/
|
||||||
.gradle-cursor
|
.gradle-cursor
|
||||||
|
App_Resources/Android/src/main/jniLibs/**/*.so
|
||||||
|
App_Resources/Android/src/main/assets/faceclaw-voice-asr/
|
||||||
|
|
||||||
# Visual Studio Code
|
# Visual Studio Code
|
||||||
.vscode/*
|
.vscode/*
|
||||||
|
|||||||
@@ -7,8 +7,10 @@ dependencies {
|
|||||||
|
|
||||||
def faceclawSherpaOnnxVersion = "1.12.34"
|
def faceclawSherpaOnnxVersion = "1.12.34"
|
||||||
def faceclawLibLc3Version = "1.1.3"
|
def faceclawLibLc3Version = "1.1.3"
|
||||||
|
def faceclawAsrModelName = "sherpa-onnx-streaming-zipformer-en-2023-06-26"
|
||||||
def faceclawAbi = "arm64-v8a"
|
def faceclawAbi = "arm64-v8a"
|
||||||
def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi")
|
def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi")
|
||||||
|
def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr")
|
||||||
def faceclawNativeBuildDir = file("$buildDir/faceclaw-native")
|
def faceclawNativeBuildDir = file("$buildDir/faceclaw-native")
|
||||||
|
|
||||||
def faceclawDownload = { String url, File destination ->
|
def faceclawDownload = { String url, File destination ->
|
||||||
@@ -56,11 +58,16 @@ tasks.register("prepareFaceclawNativeLibs") {
|
|||||||
outputs.files(
|
outputs.files(
|
||||||
file("$faceclawJniLibsDir/libonnxruntime.so"),
|
file("$faceclawJniLibsDir/libonnxruntime.so"),
|
||||||
file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"),
|
file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"),
|
||||||
file("$faceclawJniLibsDir/libfaceclaw_lc3.so")
|
file("$faceclawJniLibsDir/libfaceclaw_lc3.so"),
|
||||||
|
file("$faceclawAssetsDir/$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx"),
|
||||||
|
file("$faceclawAssetsDir/$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx"),
|
||||||
|
file("$faceclawAssetsDir/$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx"),
|
||||||
|
file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt")
|
||||||
)
|
)
|
||||||
|
|
||||||
doLast {
|
doLast {
|
||||||
faceclawJniLibsDir.mkdirs()
|
faceclawJniLibsDir.mkdirs()
|
||||||
|
faceclawAssetsDir.mkdirs()
|
||||||
|
|
||||||
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
|
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
|
||||||
def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2")
|
def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2")
|
||||||
@@ -77,6 +84,21 @@ tasks.register("prepareFaceclawNativeLibs") {
|
|||||||
into faceclawJniLibsDir
|
into faceclawJniLibsDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2")
|
||||||
|
faceclawDownload(
|
||||||
|
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2",
|
||||||
|
asrArchive
|
||||||
|
)
|
||||||
|
copy {
|
||||||
|
from tarTree(resources.bzip2(asrArchive))
|
||||||
|
include "$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx"
|
||||||
|
include "$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx"
|
||||||
|
include "$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx"
|
||||||
|
include "$faceclawAsrModelName/tokens.txt"
|
||||||
|
includeEmptyDirs = false
|
||||||
|
into faceclawAssetsDir
|
||||||
|
}
|
||||||
|
|
||||||
def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz")
|
def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz")
|
||||||
faceclawDownload(
|
faceclawDownload(
|
||||||
"https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz",
|
"https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz",
|
||||||
|
|||||||
+165
-15
@@ -12,6 +12,9 @@ import com.k2fsa.sherpa.onnx.KeywordSpotter;
|
|||||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
||||||
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
|
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
|
||||||
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
|
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineRecognizer;
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig;
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineRecognizerResult;
|
||||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||||
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
|
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
|
||||||
|
|
||||||
@@ -31,6 +34,8 @@ public class FaceclawVoiceController {
|
|||||||
private static final int STATS_INTERVAL_MS = 5_000;
|
private static final int STATS_INTERVAL_MS = 5_000;
|
||||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||||
|
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
|
||||||
|
private static final String ASR_MODEL_DIR = "sherpa-onnx-streaming-zipformer-en-2023-06-26";
|
||||||
private static final String[] MODEL_FILES = {
|
private static final String[] MODEL_FILES = {
|
||||||
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||||
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||||
@@ -38,6 +43,17 @@ public class FaceclawVoiceController {
|
|||||||
"tokens.txt",
|
"tokens.txt",
|
||||||
"screen-on-keywords.txt"
|
"screen-on-keywords.txt"
|
||||||
};
|
};
|
||||||
|
private static final String[] ASR_MODEL_FILES = {
|
||||||
|
"encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
|
||||||
|
"decoder-epoch-99-avg-1-chunk-16-left-128.onnx",
|
||||||
|
"joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
|
||||||
|
"tokens.txt"
|
||||||
|
};
|
||||||
|
|
||||||
|
private enum VoiceInputMode {
|
||||||
|
WAKEWORD,
|
||||||
|
FULL
|
||||||
|
}
|
||||||
|
|
||||||
private final Context appContext;
|
private final Context appContext;
|
||||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||||
@@ -48,9 +64,12 @@ public class FaceclawVoiceController {
|
|||||||
private volatile FaceclawBleCommunicator communicator;
|
private volatile FaceclawBleCommunicator communicator;
|
||||||
private Thread workerThread;
|
private Thread workerThread;
|
||||||
private volatile boolean started;
|
private volatile boolean started;
|
||||||
|
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
|
||||||
private KeywordSpotter keywordSpotter;
|
private KeywordSpotter keywordSpotter;
|
||||||
|
private OnlineRecognizer recognizer;
|
||||||
private OnlineStream stream;
|
private OnlineStream stream;
|
||||||
private FaceclawLc3Decoder lc3Decoder;
|
private FaceclawLc3Decoder lc3Decoder;
|
||||||
|
private String lastTranscript = "";
|
||||||
private long queuedPackets;
|
private long queuedPackets;
|
||||||
private long queueDroppedPackets;
|
private long queueDroppedPackets;
|
||||||
private long decodedSamples;
|
private long decodedSamples;
|
||||||
@@ -73,6 +92,10 @@ public class FaceclawVoiceController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void start() {
|
public void start() {
|
||||||
|
start("wakeword");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start(String requestedMode) {
|
||||||
synchronized (lock) {
|
synchronized (lock) {
|
||||||
if (started) {
|
if (started) {
|
||||||
emitStatus("Voice control is already listening.");
|
emitStatus("Voice control is already listening.");
|
||||||
@@ -82,6 +105,7 @@ public class FaceclawVoiceController {
|
|||||||
emitStatus("Voice control needs an active G2 connection.");
|
emitStatus("Voice control needs an active G2 connection.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
mode = parseMode(requestedMode);
|
||||||
started = true;
|
started = true;
|
||||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||||
workerThread.start();
|
workerThread.start();
|
||||||
@@ -103,6 +127,13 @@ public class FaceclawVoiceController {
|
|||||||
}
|
}
|
||||||
if (threadToJoin != null) {
|
if (threadToJoin != null) {
|
||||||
threadToJoin.interrupt();
|
threadToJoin.interrupt();
|
||||||
|
if (Thread.currentThread() != threadToJoin) {
|
||||||
|
try {
|
||||||
|
threadToJoin.join(1500);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,19 +141,35 @@ public class FaceclawVoiceController {
|
|||||||
stop();
|
stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private VoiceInputMode parseMode(String requestedMode) {
|
||||||
|
return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD;
|
||||||
|
}
|
||||||
|
|
||||||
private void runLoop() {
|
private void runLoop() {
|
||||||
try {
|
try {
|
||||||
emitStatus("Voice control loading wake-word model...");
|
VoiceInputMode currentMode = mode;
|
||||||
File modelDir = installModelFiles();
|
emitStatus(currentMode == VoiceInputMode.FULL
|
||||||
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
? "Voice control loading transcription model..."
|
||||||
stream = keywordSpotter.createStream();
|
: "Voice control loading wake-word model...");
|
||||||
|
if (currentMode == VoiceInputMode.FULL) {
|
||||||
|
File modelDir = installAsrModelFiles();
|
||||||
|
recognizer = new OnlineRecognizer(buildRecognizerConfig(modelDir));
|
||||||
|
stream = recognizer.createStream();
|
||||||
|
lastTranscript = "";
|
||||||
|
} else {
|
||||||
|
File modelDir = installModelFiles();
|
||||||
|
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
||||||
|
stream = keywordSpotter.createStream();
|
||||||
|
}
|
||||||
lc3Decoder = new FaceclawLc3Decoder();
|
lc3Decoder = new FaceclawLc3Decoder();
|
||||||
if (!startG2Audio()) {
|
if (!startG2Audio()) {
|
||||||
emitStatus("Voice control could not start G2 microphone input.");
|
emitStatus("Voice control could not start G2 microphone input.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
emitStatus("Voice control listening for \"screen on\" from the G2 mic.");
|
emitStatus(currentMode == VoiceInputMode.FULL
|
||||||
|
? "Voice control transcribing from the G2 mic."
|
||||||
|
: "Voice control listening for \"screen on\" from the G2 mic.");
|
||||||
processG2Audio();
|
processG2Audio();
|
||||||
} catch (Throwable error) {
|
} catch (Throwable error) {
|
||||||
Log.e(TAG, "Voice control failed", error);
|
Log.e(TAG, "Voice control failed", error);
|
||||||
@@ -152,6 +199,7 @@ public class FaceclawVoiceController {
|
|||||||
.build())
|
.build())
|
||||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||||
.setModelType("zipformer2")
|
.setModelType("zipformer2")
|
||||||
|
.setModelingUnit("")
|
||||||
.setNumThreads(1)
|
.setNumThreads(1)
|
||||||
.build())
|
.build())
|
||||||
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
|
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
|
||||||
@@ -160,6 +208,26 @@ public class FaceclawVoiceController {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private OnlineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||||
|
return OnlineRecognizerConfig.builder()
|
||||||
|
.setFeatureConfig(FeatureConfig.builder()
|
||||||
|
.setSampleRate(SAMPLE_RATE)
|
||||||
|
.setFeatureDim(FEATURE_DIM)
|
||||||
|
.build())
|
||||||
|
.setOnlineModelConfig(OnlineModelConfig.builder()
|
||||||
|
.setTransducer(OnlineTransducerModelConfig.builder()
|
||||||
|
.setEncoder(new File(modelDir, "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
|
||||||
|
.setDecoder(new File(modelDir, "decoder-epoch-99-avg-1-chunk-16-left-128.onnx").getAbsolutePath())
|
||||||
|
.setJoiner(new File(modelDir, "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
|
||||||
|
.build())
|
||||||
|
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||||
|
.setModelType("zipformer2")
|
||||||
|
.setNumThreads(1)
|
||||||
|
.build())
|
||||||
|
.setEnableEndpoint(true)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
private File installModelFiles() throws IOException {
|
private File installModelFiles() throws IOException {
|
||||||
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
|
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
|
||||||
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||||
@@ -176,6 +244,22 @@ public class FaceclawVoiceController {
|
|||||||
return modelDir;
|
return modelDir;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private File installAsrModelFiles() throws IOException {
|
||||||
|
File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR);
|
||||||
|
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||||
|
throw new IOException("Could not create " + modelDir.getAbsolutePath());
|
||||||
|
}
|
||||||
|
AssetManager assets = appContext.getAssets();
|
||||||
|
for (String fileName : ASR_MODEL_FILES) {
|
||||||
|
copyAssetIfNeeded(
|
||||||
|
assets,
|
||||||
|
ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName,
|
||||||
|
new File(modelDir, fileName)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return modelDir;
|
||||||
|
}
|
||||||
|
|
||||||
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
|
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
|
||||||
if (destination.exists() && destination.length() > 0) {
|
if (destination.exists() && destination.length() > 0) {
|
||||||
return;
|
return;
|
||||||
@@ -206,9 +290,8 @@ public class FaceclawVoiceController {
|
|||||||
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
|
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
|
||||||
while (started && !Thread.currentThread().isInterrupted()) {
|
while (started && !Thread.currentThread().isInterrupted()) {
|
||||||
OnlineStream currentStream = stream;
|
OnlineStream currentStream = stream;
|
||||||
KeywordSpotter currentSpotter = keywordSpotter;
|
|
||||||
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
||||||
if (currentStream == null || currentSpotter == null || currentDecoder == null) {
|
if (currentStream == null || currentDecoder == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,19 +312,73 @@ public class FaceclawVoiceController {
|
|||||||
}
|
}
|
||||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||||
|
|
||||||
while (currentSpotter.isReady(currentStream)) {
|
if (mode == VoiceInputMode.FULL) {
|
||||||
currentSpotter.decode(currentStream);
|
processRecognizer(currentStream);
|
||||||
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
|
} else {
|
||||||
String keyword = result == null ? "" : result.getKeyword();
|
processKeywordSpotter(currentStream);
|
||||||
if (keyword != null && keyword.trim().length() > 0) {
|
|
||||||
currentSpotter.reset(currentStream);
|
|
||||||
emitWakeWord(keyword);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
maybeEmitAudioStats(false);
|
maybeEmitAudioStats(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void processKeywordSpotter(OnlineStream currentStream) {
|
||||||
|
KeywordSpotter currentSpotter = keywordSpotter;
|
||||||
|
if (currentSpotter == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (currentSpotter.isReady(currentStream)) {
|
||||||
|
currentSpotter.decode(currentStream);
|
||||||
|
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
|
||||||
|
String keyword = result == null ? "" : result.getKeyword();
|
||||||
|
if (keyword != null && keyword.trim().length() > 0) {
|
||||||
|
currentSpotter.reset(currentStream);
|
||||||
|
emitWakeWord(keyword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processRecognizer(OnlineStream currentStream) {
|
||||||
|
OnlineRecognizer currentRecognizer = recognizer;
|
||||||
|
if (currentRecognizer == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (currentRecognizer.isReady(currentStream)) {
|
||||||
|
currentRecognizer.decode(currentStream);
|
||||||
|
}
|
||||||
|
emitRecognizerResult(currentRecognizer, currentStream, false);
|
||||||
|
if (currentRecognizer.isEndpoint(currentStream)) {
|
||||||
|
emitRecognizerResult(currentRecognizer, currentStream, true);
|
||||||
|
currentRecognizer.reset(currentStream);
|
||||||
|
lastTranscript = "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void emitRecognizerResult(OnlineRecognizer currentRecognizer, OnlineStream currentStream, boolean isFinal) {
|
||||||
|
OnlineRecognizerResult result = currentRecognizer.getResult(currentStream);
|
||||||
|
String text = result == null ? "" : result.getText();
|
||||||
|
if (text == null) {
|
||||||
|
text = "";
|
||||||
|
}
|
||||||
|
if (text.equals(lastTranscript)) {
|
||||||
|
if (isFinal && lastTranscript.length() > 0) {
|
||||||
|
emitTranscript("", true);
|
||||||
|
lastTranscript = "";
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String delta = text;
|
||||||
|
if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) {
|
||||||
|
delta = text.substring(lastTranscript.length());
|
||||||
|
}
|
||||||
|
if (delta.trim().length() > 0) {
|
||||||
|
emitTranscript(delta, false);
|
||||||
|
}
|
||||||
|
lastTranscript = isFinal ? "" : text;
|
||||||
|
if (isFinal) {
|
||||||
|
emitTranscript("", true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void stopG2Audio() {
|
private void stopG2Audio() {
|
||||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||||
if (currentCommunicator != null) {
|
if (currentCommunicator != null) {
|
||||||
@@ -259,6 +396,10 @@ public class FaceclawVoiceController {
|
|||||||
keywordSpotter.release();
|
keywordSpotter.release();
|
||||||
keywordSpotter = null;
|
keywordSpotter = null;
|
||||||
}
|
}
|
||||||
|
if (recognizer != null) {
|
||||||
|
recognizer.release();
|
||||||
|
recognizer = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseLc3() {
|
private void releaseLc3() {
|
||||||
@@ -339,6 +480,7 @@ public class FaceclawVoiceController {
|
|||||||
+ " duplicate=" + duplicate
|
+ " duplicate=" + duplicate
|
||||||
+ " late=" + latePackets
|
+ " late=" + latePackets
|
||||||
+ " maxGapMs=" + maxInterPacketMs
|
+ " maxGapMs=" + maxInterPacketMs
|
||||||
|
+ "\n"
|
||||||
+ " queueDrop=" + queueDroppedPackets
|
+ " queueDrop=" + queueDroppedPackets
|
||||||
+ " decodeErrors=" + decodeErrors
|
+ " decodeErrors=" + decodeErrors
|
||||||
+ " wrongArm=" + wrongArmPackets
|
+ " wrongArm=" + wrongArmPackets
|
||||||
@@ -363,6 +505,14 @@ public class FaceclawVoiceController {
|
|||||||
mainHandler.post(() -> currentListener.onWakeWord(keyword));
|
mainHandler.post(() -> currentListener.onWakeWord(keyword));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void emitTranscript(String text, boolean isFinal) {
|
||||||
|
FaceclawVoiceControllerListener currentListener = listener;
|
||||||
|
if (currentListener == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
mainHandler.post(() -> currentListener.onTranscript(text, isFinal));
|
||||||
|
}
|
||||||
|
|
||||||
private static final class AudioPacket {
|
private static final class AudioPacket {
|
||||||
final byte[] data;
|
final byte[] data;
|
||||||
final String arm;
|
final String arm;
|
||||||
|
|||||||
+2
@@ -4,4 +4,6 @@ public interface FaceclawVoiceControllerListener {
|
|||||||
void onStatus(String status);
|
void onStatus(String status);
|
||||||
|
|
||||||
void onWakeWord(String keyword);
|
void onWakeWord(String keyword);
|
||||||
|
|
||||||
|
void onTranscript(String text, boolean isFinal);
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class EndpointConfig {
|
||||||
|
private final EndpointRule rule1;
|
||||||
|
private final EndpointRule rule2;
|
||||||
|
private final EndpointRule rule3;
|
||||||
|
|
||||||
|
private EndpointConfig(Builder builder) {
|
||||||
|
this.rule1 = builder.rule1;
|
||||||
|
this.rule2 = builder.rule2;
|
||||||
|
this.rule3 = builder.rule3;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointRule getRule1() {
|
||||||
|
return rule1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointRule getRule2() {
|
||||||
|
return rule2;
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointRule getRule3() {
|
||||||
|
return rule3;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private EndpointRule rule1 = EndpointRule.builder()
|
||||||
|
.setMustContainNonSilence(false)
|
||||||
|
.setMinTrailingSilence(2.4f)
|
||||||
|
.setMinUtteranceLength(0.0f)
|
||||||
|
.build();
|
||||||
|
private EndpointRule rule2 = EndpointRule.builder()
|
||||||
|
.setMustContainNonSilence(true)
|
||||||
|
.setMinTrailingSilence(1.2f)
|
||||||
|
.setMinUtteranceLength(0.0f)
|
||||||
|
.build();
|
||||||
|
private EndpointRule rule3 = EndpointRule.builder()
|
||||||
|
.setMustContainNonSilence(false)
|
||||||
|
.setMinTrailingSilence(0.0f)
|
||||||
|
.setMinUtteranceLength(20.0f)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
public Builder setRule1(EndpointRule rule1) {
|
||||||
|
this.rule1 = rule1;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setRule2(EndpointRule rule2) {
|
||||||
|
this.rule2 = rule2;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setRule3(EndpointRule rule3) {
|
||||||
|
this.rule3 = rule3;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointConfig build() {
|
||||||
|
return new EndpointConfig(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class EndpointRule {
|
||||||
|
private final boolean mustContainNonSilence;
|
||||||
|
private final float minTrailingSilence;
|
||||||
|
private final float minUtteranceLength;
|
||||||
|
|
||||||
|
private EndpointRule(Builder builder) {
|
||||||
|
this.mustContainNonSilence = builder.mustContainNonSilence;
|
||||||
|
this.minTrailingSilence = builder.minTrailingSilence;
|
||||||
|
this.minUtteranceLength = builder.minUtteranceLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean getMustContainNonSilence() {
|
||||||
|
return mustContainNonSilence;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getMinTrailingSilence() {
|
||||||
|
return minTrailingSilence;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getMinUtteranceLength() {
|
||||||
|
return minUtteranceLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private boolean mustContainNonSilence = false;
|
||||||
|
private float minTrailingSilence = 0.0f;
|
||||||
|
private float minUtteranceLength = 0.0f;
|
||||||
|
|
||||||
|
public Builder setMustContainNonSilence(boolean mustContainNonSilence) {
|
||||||
|
this.mustContainNonSilence = mustContainNonSilence;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setMinTrailingSilence(float minTrailingSilence) {
|
||||||
|
this.minTrailingSilence = minTrailingSilence;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setMinUtteranceLength(float minUtteranceLength) {
|
||||||
|
this.minUtteranceLength = minUtteranceLength;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointRule build() {
|
||||||
|
return new EndpointRule(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+54
@@ -0,0 +1,54 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class HomophoneReplacerConfig {
|
||||||
|
private final String dictDir;
|
||||||
|
private final String lexicon;
|
||||||
|
private final String ruleFsts;
|
||||||
|
|
||||||
|
private HomophoneReplacerConfig(Builder builder) {
|
||||||
|
this.dictDir = builder.dictDir;
|
||||||
|
this.lexicon = builder.lexicon;
|
||||||
|
this.ruleFsts = builder.ruleFsts;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDictDir() {
|
||||||
|
return dictDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLexicon() {
|
||||||
|
return lexicon;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRuleFsts() {
|
||||||
|
return ruleFsts;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private String dictDir = "";
|
||||||
|
private String lexicon = "";
|
||||||
|
private String ruleFsts = "";
|
||||||
|
|
||||||
|
public Builder setDictDir(String dictDir) {
|
||||||
|
this.dictDir = dictDir;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setLexicon(String lexicon) {
|
||||||
|
this.lexicon = lexicon;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setRuleFsts(String ruleFsts) {
|
||||||
|
this.ruleFsts = ruleFsts;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public HomophoneReplacerConfig build() {
|
||||||
|
return new HomophoneReplacerConfig(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+42
@@ -0,0 +1,42 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class OnlineCtcFstDecoderConfig {
|
||||||
|
private final String graph;
|
||||||
|
private final int maxActive;
|
||||||
|
|
||||||
|
private OnlineCtcFstDecoderConfig(Builder builder) {
|
||||||
|
this.graph = builder.graph;
|
||||||
|
this.maxActive = builder.maxActive;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGraph() {
|
||||||
|
return graph;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getMaxActive() {
|
||||||
|
return maxActive;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private String graph = "";
|
||||||
|
private int maxActive = 3000;
|
||||||
|
|
||||||
|
public Builder setGraph(String graph) {
|
||||||
|
this.graph = graph;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setMaxActive(int maxActive) {
|
||||||
|
this.maxActive = maxActive;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineCtcFstDecoderConfig build() {
|
||||||
|
return new OnlineCtcFstDecoderConfig(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class OnlineLMConfig {
|
||||||
|
private final String model;
|
||||||
|
private final float scale;
|
||||||
|
|
||||||
|
private OnlineLMConfig(Builder builder) {
|
||||||
|
this.model = builder.model;
|
||||||
|
this.scale = builder.scale;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getModel() {
|
||||||
|
return model;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getScale() {
|
||||||
|
return scale;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private String model = "";
|
||||||
|
private float scale = 0.5f;
|
||||||
|
|
||||||
|
public Builder setModel(String model) {
|
||||||
|
this.model = model;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setScale(float scale) {
|
||||||
|
this.scale = scale;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineLMConfig build() {
|
||||||
|
return new OnlineLMConfig(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class OnlineRecognizer {
|
||||||
|
private long ptr;
|
||||||
|
|
||||||
|
public OnlineRecognizer(OnlineRecognizerConfig config) {
|
||||||
|
System.loadLibrary("onnxruntime");
|
||||||
|
System.loadLibrary("sherpa-onnx-jni");
|
||||||
|
ptr = newFromFile(config);
|
||||||
|
if (ptr == 0) {
|
||||||
|
throw new IllegalArgumentException("Invalid OnlineRecognizerConfig");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineStream createStream() {
|
||||||
|
return createStream("");
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineStream createStream(String hotwords) {
|
||||||
|
long streamPtr = createStream(ptr, hotwords == null ? "" : hotwords);
|
||||||
|
return new OnlineStream(streamPtr);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isReady(OnlineStream stream) {
|
||||||
|
return stream != null && isReady(ptr, stream.getPtr());
|
||||||
|
}
|
||||||
|
|
||||||
|
public void decode(OnlineStream stream) {
|
||||||
|
if (stream != null) {
|
||||||
|
decode(ptr, stream.getPtr());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reset(OnlineStream stream) {
|
||||||
|
if (stream != null) {
|
||||||
|
reset(ptr, stream.getPtr());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isEndpoint(OnlineStream stream) {
|
||||||
|
return stream != null && isEndpoint(ptr, stream.getPtr());
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineRecognizerResult getResult(OnlineStream stream) {
|
||||||
|
if (stream == null) {
|
||||||
|
return new OnlineRecognizerResult("", new String[0], new float[0], new float[0]);
|
||||||
|
}
|
||||||
|
return getResult(ptr, stream.getPtr());
|
||||||
|
}
|
||||||
|
|
||||||
|
public void release() {
|
||||||
|
if (ptr != 0) {
|
||||||
|
delete(ptr);
|
||||||
|
ptr = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private native long newFromFile(OnlineRecognizerConfig config);
|
||||||
|
private native void delete(long ptr);
|
||||||
|
private native long createStream(long ptr, String hotwords);
|
||||||
|
private native void reset(long ptr, long streamPtr);
|
||||||
|
private native void decode(long ptr, long streamPtr);
|
||||||
|
private native boolean isEndpoint(long ptr, long streamPtr);
|
||||||
|
private native boolean isReady(long ptr, long streamPtr);
|
||||||
|
private native OnlineRecognizerResult getResult(long ptr, long streamPtr);
|
||||||
|
}
|
||||||
+146
@@ -0,0 +1,146 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class OnlineRecognizerConfig {
|
||||||
|
private final FeatureConfig featConfig;
|
||||||
|
private final OnlineModelConfig modelConfig;
|
||||||
|
private final OnlineLMConfig lmConfig;
|
||||||
|
private final OnlineCtcFstDecoderConfig ctcFstDecoderConfig;
|
||||||
|
private final HomophoneReplacerConfig hr;
|
||||||
|
private final EndpointConfig endpointConfig;
|
||||||
|
private final boolean enableEndpoint;
|
||||||
|
private final String decodingMethod;
|
||||||
|
private final int maxActivePaths;
|
||||||
|
private final String hotwordsFile;
|
||||||
|
private final float hotwordsScore;
|
||||||
|
private final String ruleFsts;
|
||||||
|
private final String ruleFars;
|
||||||
|
private final float blankPenalty;
|
||||||
|
|
||||||
|
private OnlineRecognizerConfig(Builder builder) {
|
||||||
|
this.featConfig = builder.featConfig;
|
||||||
|
this.modelConfig = builder.modelConfig;
|
||||||
|
this.lmConfig = builder.lmConfig;
|
||||||
|
this.ctcFstDecoderConfig = builder.ctcFstDecoderConfig;
|
||||||
|
this.hr = builder.hr;
|
||||||
|
this.endpointConfig = builder.endpointConfig;
|
||||||
|
this.enableEndpoint = builder.enableEndpoint;
|
||||||
|
this.decodingMethod = builder.decodingMethod;
|
||||||
|
this.maxActivePaths = builder.maxActivePaths;
|
||||||
|
this.hotwordsFile = builder.hotwordsFile;
|
||||||
|
this.hotwordsScore = builder.hotwordsScore;
|
||||||
|
this.ruleFsts = builder.ruleFsts;
|
||||||
|
this.ruleFars = builder.ruleFars;
|
||||||
|
this.blankPenalty = builder.blankPenalty;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Builder builder() {
|
||||||
|
return new Builder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public FeatureConfig getFeatConfig() {
|
||||||
|
return featConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineModelConfig getModelConfig() {
|
||||||
|
return modelConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineLMConfig getLmConfig() {
|
||||||
|
return lmConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineCtcFstDecoderConfig getCtcFstDecoderConfig() {
|
||||||
|
return ctcFstDecoderConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
public HomophoneReplacerConfig getHr() {
|
||||||
|
return hr;
|
||||||
|
}
|
||||||
|
|
||||||
|
public EndpointConfig getEndpointConfig() {
|
||||||
|
return endpointConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean getEnableEndpoint() {
|
||||||
|
return enableEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDecodingMethod() {
|
||||||
|
return decodingMethod;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getMaxActivePaths() {
|
||||||
|
return maxActivePaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getHotwordsFile() {
|
||||||
|
return hotwordsFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getHotwordsScore() {
|
||||||
|
return hotwordsScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRuleFsts() {
|
||||||
|
return ruleFsts;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRuleFars() {
|
||||||
|
return ruleFars;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getBlankPenalty() {
|
||||||
|
return blankPenalty;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Builder {
|
||||||
|
private FeatureConfig featConfig = FeatureConfig.builder().build();
|
||||||
|
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
|
||||||
|
private OnlineLMConfig lmConfig = OnlineLMConfig.builder().build();
|
||||||
|
private OnlineCtcFstDecoderConfig ctcFstDecoderConfig = OnlineCtcFstDecoderConfig.builder().build();
|
||||||
|
private HomophoneReplacerConfig hr = HomophoneReplacerConfig.builder().build();
|
||||||
|
private EndpointConfig endpointConfig = EndpointConfig.builder().build();
|
||||||
|
private boolean enableEndpoint = true;
|
||||||
|
private String decodingMethod = "greedy_search";
|
||||||
|
private int maxActivePaths = 4;
|
||||||
|
private String hotwordsFile = "";
|
||||||
|
private float hotwordsScore = 1.5f;
|
||||||
|
private String ruleFsts = "";
|
||||||
|
private String ruleFars = "";
|
||||||
|
private float blankPenalty = 0.0f;
|
||||||
|
|
||||||
|
public Builder setFeatureConfig(FeatureConfig featConfig) {
|
||||||
|
this.featConfig = featConfig;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
|
||||||
|
this.modelConfig = modelConfig;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setEndpointConfig(EndpointConfig endpointConfig) {
|
||||||
|
this.endpointConfig = endpointConfig;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setEnableEndpoint(boolean enableEndpoint) {
|
||||||
|
this.enableEndpoint = enableEndpoint;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setDecodingMethod(String decodingMethod) {
|
||||||
|
this.decodingMethod = decodingMethod;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder setMaxActivePaths(int maxActivePaths) {
|
||||||
|
this.maxActivePaths = maxActivePaths;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnlineRecognizerConfig build() {
|
||||||
|
return new OnlineRecognizerConfig(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package com.k2fsa.sherpa.onnx;
|
||||||
|
|
||||||
|
public class OnlineRecognizerResult {
|
||||||
|
private final String text;
|
||||||
|
private final String[] tokens;
|
||||||
|
private final float[] timestamps;
|
||||||
|
private final float[] ysProbs;
|
||||||
|
|
||||||
|
public OnlineRecognizerResult(String text, String[] tokens, float[] timestamps, float[] ysProbs) {
|
||||||
|
this.text = text == null ? "" : text;
|
||||||
|
this.tokens = tokens == null ? new String[0] : tokens;
|
||||||
|
this.timestamps = timestamps == null ? new float[0] : timestamps;
|
||||||
|
this.ysProbs = ysProbs == null ? new float[0] : ysProbs;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getText() {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] getTokens() {
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float[] getTimestamps() {
|
||||||
|
return timestamps;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float[] getYsProbs() {
|
||||||
|
return ysProbs;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -52,6 +52,7 @@ const CONTAINER_NAME = "dashboard";
|
|||||||
const DASHBOARD_INTERVAL_MS = 60_000;
|
const DASHBOARD_INTERVAL_MS = 60_000;
|
||||||
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
|
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
|
||||||
const STOPWATCH_RENDER_INTERVAL_MS = 100;
|
const STOPWATCH_RENDER_INTERVAL_MS = 100;
|
||||||
|
const TRANSCRIBE_RENDER_INTERVAL_MS = 250;
|
||||||
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
|
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
|
||||||
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
|
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
|
||||||
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
|
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
|
||||||
@@ -120,6 +121,7 @@ class DashboardController {
|
|||||||
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
|
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
|
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private stopwatchRenderTimer: ReturnType<typeof setInterval> | null = null;
|
private stopwatchRenderTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
private transcribeRenderTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private offState: (() => void) | null = null;
|
private offState: (() => void) | null = null;
|
||||||
private offLog: (() => void) | null = null;
|
private offLog: (() => void) | null = null;
|
||||||
private offRing: (() => void) | null = null;
|
private offRing: (() => void) | null = null;
|
||||||
@@ -149,6 +151,9 @@ class DashboardController {
|
|||||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||||
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
|
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
|
||||||
setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active),
|
setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active),
|
||||||
|
setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active),
|
||||||
|
startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode),
|
||||||
|
stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -393,6 +398,10 @@ class DashboardController {
|
|||||||
this.offVoiceStatus = null;
|
this.offVoiceStatus = null;
|
||||||
this.offVoiceWakeWord?.();
|
this.offVoiceWakeWord?.();
|
||||||
this.offVoiceWakeWord = null;
|
this.offVoiceWakeWord = null;
|
||||||
|
if (this.transcribeRenderTimer) {
|
||||||
|
clearInterval(this.transcribeRenderTimer);
|
||||||
|
this.transcribeRenderTimer = null;
|
||||||
|
}
|
||||||
await mediaControllerBridge.stop().catch(() => {});
|
await mediaControllerBridge.stop().catch(() => {});
|
||||||
await nightscoutBridge.stop().catch(() => {});
|
await nightscoutBridge.stop().catch(() => {});
|
||||||
voiceControlBridge.stop();
|
voiceControlBridge.stop();
|
||||||
@@ -452,6 +461,10 @@ class DashboardController {
|
|||||||
await mediaControllerBridge.stop().catch(() => {});
|
await mediaControllerBridge.stop().catch(() => {});
|
||||||
await nightscoutBridge.stop().catch(() => {});
|
await nightscoutBridge.stop().catch(() => {});
|
||||||
voiceControlBridge.stop();
|
voiceControlBridge.stop();
|
||||||
|
if (this.transcribeRenderTimer) {
|
||||||
|
clearInterval(this.transcribeRenderTimer);
|
||||||
|
this.transcribeRenderTimer = null;
|
||||||
|
}
|
||||||
await communicator?.close().catch(() => {});
|
await communicator?.close().catch(() => {});
|
||||||
} finally {
|
} finally {
|
||||||
stopForegroundNotification();
|
stopForegroundNotification();
|
||||||
@@ -673,7 +686,21 @@ class DashboardController {
|
|||||||
if (!this.communicator) return;
|
if (!this.communicator) return;
|
||||||
if (!isDashboardVoiceControlEnabled()) return;
|
if (!isDashboardVoiceControlEnabled()) return;
|
||||||
this.updateConnectedForegroundNotification();
|
this.updateConnectedForegroundNotification();
|
||||||
voiceControlBridge.start(this.communicator.getNativeCommunicator());
|
voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise<void> {
|
||||||
|
if (!this.communicator) {
|
||||||
|
throw new Error("Voice input needs an active G2 connection.");
|
||||||
|
}
|
||||||
|
await ensureVoicePermissions();
|
||||||
|
this.updateConnectedForegroundNotification();
|
||||||
|
voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private stopDedicatedVoiceInput(): void {
|
||||||
|
voiceControlBridge.stop();
|
||||||
|
this.startVoiceControlIfEnabled();
|
||||||
}
|
}
|
||||||
|
|
||||||
private setStopwatchRenderActive(active: boolean): void {
|
private setStopwatchRenderActive(active: boolean): void {
|
||||||
@@ -692,6 +719,22 @@ class DashboardController {
|
|||||||
}, STOPWATCH_RENDER_INTERVAL_MS);
|
}, STOPWATCH_RENDER_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private setTranscribeRenderActive(active: boolean): void {
|
||||||
|
if (!active || this.phase !== "connected" || !this.communicator) {
|
||||||
|
if (this.transcribeRenderTimer) {
|
||||||
|
clearInterval(this.transcribeRenderTimer);
|
||||||
|
this.transcribeRenderTimer = null;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.transcribeRenderTimer) return;
|
||||||
|
this.transcribeRenderTimer = setInterval(() => {
|
||||||
|
void this.requestRender("interval").catch((error) => {
|
||||||
|
this.appendLog(`transcribe render failed: ${this.formatError(error)}`);
|
||||||
|
});
|
||||||
|
}, TRANSCRIBE_RENDER_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
private clearDashboardTimer(): void {
|
private clearDashboardTimer(): void {
|
||||||
if (this.dashboardTimer) {
|
if (this.dashboardTimer) {
|
||||||
clearInterval(this.dashboardTimer);
|
clearInterval(this.dashboardTimer);
|
||||||
@@ -705,6 +748,10 @@ class DashboardController {
|
|||||||
clearInterval(this.stopwatchRenderTimer);
|
clearInterval(this.stopwatchRenderTimer);
|
||||||
this.stopwatchRenderTimer = null;
|
this.stopwatchRenderTimer = null;
|
||||||
}
|
}
|
||||||
|
if (this.transcribeRenderTimer) {
|
||||||
|
clearInterval(this.transcribeRenderTimer);
|
||||||
|
this.transcribeRenderTimer = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private setPhase(phase: ConnectionPhase): void {
|
private setPhase(phase: ConnectionPhase): void {
|
||||||
|
|||||||
@@ -6,13 +6,21 @@ export type VoiceControlState = {
|
|||||||
status: string;
|
status: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type VoiceInputMode = "off" | "wakeword" | "full";
|
||||||
|
export type VoiceTranscriptEvent = {
|
||||||
|
text: string;
|
||||||
|
isFinal: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export class FaceclawVoiceControlBridge {
|
export class FaceclawVoiceControlBridge {
|
||||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||||
|
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||||
private controller: any | null = null;
|
private controller: any | null = null;
|
||||||
private listenerProxy: any | null = null;
|
private listenerProxy: any | null = null;
|
||||||
private status = "Voice control stopped.";
|
private status = "Voice control stopped.";
|
||||||
private started = false;
|
private started = false;
|
||||||
|
private mode: VoiceInputMode = "off";
|
||||||
|
|
||||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||||
this.statusListeners.add(listener);
|
this.statusListeners.add(listener);
|
||||||
@@ -25,19 +33,34 @@ export class FaceclawVoiceControlBridge {
|
|||||||
return () => this.wakeWordListeners.delete(listener);
|
return () => this.wakeWordListeners.delete(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
start(communicator?: any): void {
|
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||||
if (this.started || !global.isAndroid) return;
|
this.transcriptListeners.add(listener);
|
||||||
|
return () => this.transcriptListeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
start(communicator?: any, mode: VoiceInputMode = "wakeword"): void {
|
||||||
|
if (mode === "off") {
|
||||||
|
this.stop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!global.isAndroid) return;
|
||||||
this.ensureController();
|
this.ensureController();
|
||||||
if (communicator) {
|
if (communicator) {
|
||||||
this.controller?.setCommunicator(communicator);
|
this.controller?.setCommunicator(communicator);
|
||||||
}
|
}
|
||||||
|
if (this.started && this.mode !== mode) {
|
||||||
|
this.controller?.stop();
|
||||||
|
this.started = false;
|
||||||
|
}
|
||||||
this.started = true;
|
this.started = true;
|
||||||
this.controller?.start();
|
this.mode = mode;
|
||||||
|
this.controller?.start(mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
stop(): void {
|
stop(): void {
|
||||||
if (!this.started || !global.isAndroid) return;
|
if (!this.started || !global.isAndroid) return;
|
||||||
this.started = false;
|
this.started = false;
|
||||||
|
this.mode = "off";
|
||||||
this.controller?.stop();
|
this.controller?.stop();
|
||||||
this.setStatus("Voice control stopped.");
|
this.setStatus("Voice control stopped.");
|
||||||
}
|
}
|
||||||
@@ -58,6 +81,12 @@ export class FaceclawVoiceControlBridge {
|
|||||||
listener(String(keyword));
|
listener(String(keyword));
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
onTranscript: (text: string, isFinal: boolean) => {
|
||||||
|
const event = { text: String(text), isFinal: Boolean(isFinal) };
|
||||||
|
for (const listener of this.transcriptListeners) {
|
||||||
|
listener(event);
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
this.controller.setListener(this.listenerProxy);
|
this.controller.setListener(this.listenerProxy);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { nightscoutBridge } from "../native/nightscout";
|
|||||||
import { readActiveNotificationIcons } from "../native/notification-icons";
|
import { readActiveNotificationIcons } from "../native/notification-icons";
|
||||||
import { readPhoneBatteryState } from "../native/phone-battery";
|
import { readPhoneBatteryState } from "../native/phone-battery";
|
||||||
import { readSystemStatusIcons } from "../native/system-status-icons";
|
import { readSystemStatusIcons } from "../native/system-status-icons";
|
||||||
|
import { voiceControlBridge, type VoiceTranscriptEvent } from "../native/voice-control";
|
||||||
import { EventSourceType, OsEventTypeList } from "../g2/events";
|
import { EventSourceType, OsEventTypeList } from "../g2/events";
|
||||||
import {
|
import {
|
||||||
getDashboardPlugin,
|
getDashboardPlugin,
|
||||||
@@ -85,6 +86,9 @@ const dashboardActions: LayerActions = {
|
|||||||
endTextSettingEdit: () => {},
|
endTextSettingEdit: () => {},
|
||||||
setVoiceControlEnabled: () => {},
|
setVoiceControlEnabled: () => {},
|
||||||
setStopwatchRenderActive: () => {},
|
setStopwatchRenderActive: () => {},
|
||||||
|
setTranscribeRenderActive: () => {},
|
||||||
|
startDedicatedVoiceInput: () => {},
|
||||||
|
stopDedicatedVoiceInput: () => {},
|
||||||
};
|
};
|
||||||
const dashboardFont = loadEmbeddedTerminus12();
|
const dashboardFont = loadEmbeddedTerminus12();
|
||||||
const dashboardSystemFont = loadEmbeddedTerminus16();
|
const dashboardSystemFont = loadEmbeddedTerminus16();
|
||||||
@@ -487,6 +491,14 @@ function createAppsMenuLayer(): MenuLayer {
|
|||||||
void ctx.actions.setStopwatchRenderActive(true);
|
void ctx.actions.setStopwatchRenderActive(true);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
label: "Transcribe",
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
const layer = new TranscribeLayer();
|
||||||
|
ctx.stack.push(layer);
|
||||||
|
layer.start(ctx);
|
||||||
|
},
|
||||||
|
},
|
||||||
],
|
],
|
||||||
TOP_LEFT_MENU_LAYOUT,
|
TOP_LEFT_MENU_LAYOUT,
|
||||||
);
|
);
|
||||||
@@ -539,6 +551,122 @@ class StopwatchLayer implements Layer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class TranscribeLayer implements Layer {
|
||||||
|
private mode: "off" | "wakeword" | "full" = "full";
|
||||||
|
private status = "Starting transcription...";
|
||||||
|
private transcript = "";
|
||||||
|
private unsubscribeTranscript: (() => void) | null = null;
|
||||||
|
private unsubscribeStatus: (() => void) | null = null;
|
||||||
|
|
||||||
|
start(ctx: LayerContext): void {
|
||||||
|
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||||
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||||
|
this.status = state.status;
|
||||||
|
});
|
||||||
|
void ctx.actions.setTranscribeRenderActive(true);
|
||||||
|
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
|
||||||
|
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
paint(ctx: LayerContext): GrayImage {
|
||||||
|
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||||
|
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
|
||||||
|
const text = this.transcript || "Listening...";
|
||||||
|
const wrapped = wrapTranscribeText(ctx.font, text, G2_LENS_WIDTH - 64);
|
||||||
|
|
||||||
|
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
|
||||||
|
image.drawText(ctx.font, 24, 24, "Transcribe", 200);
|
||||||
|
image.drawText(ctx.font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
|
||||||
|
//image.drawText(ctx.font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
|
||||||
|
image.drawText(ctx.font, 24, 46, this.status, 110);
|
||||||
|
|
||||||
|
const firstLine = Math.max(0, wrapped.length - 10);
|
||||||
|
for (let index = firstLine; index < wrapped.length; index++) {
|
||||||
|
const y = 72 + (index - firstLine) * 16;
|
||||||
|
image.drawText(ctx.font, 32, y, wrapped[index]!, 230);
|
||||||
|
}
|
||||||
|
|
||||||
|
image.drawText(ctx.font, 24, 252, "Click: mode / Double-click: back", 110);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||||
|
if (event.type === "double-click") {
|
||||||
|
this.stop(ctx);
|
||||||
|
ctx.stack.pop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.type === "click") {
|
||||||
|
this.cycleMode(ctx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private cycleMode(ctx: LayerContext): void {
|
||||||
|
if (this.mode === "full") {
|
||||||
|
this.mode = "wakeword";
|
||||||
|
void ctx.actions.startDedicatedVoiceInput("wakeword");
|
||||||
|
} else if (this.mode === "wakeword") {
|
||||||
|
this.mode = "off";
|
||||||
|
void ctx.actions.stopDedicatedVoiceInput();
|
||||||
|
} else {
|
||||||
|
this.mode = "full";
|
||||||
|
void ctx.actions.startDedicatedVoiceInput("full");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private stop(ctx: LayerContext): void {
|
||||||
|
this.unsubscribeTranscript?.();
|
||||||
|
this.unsubscribeTranscript = null;
|
||||||
|
this.unsubscribeStatus?.();
|
||||||
|
this.unsubscribeStatus = null;
|
||||||
|
void ctx.actions.setTranscribeRenderActive(false);
|
||||||
|
void ctx.actions.stopDedicatedVoiceInput();
|
||||||
|
}
|
||||||
|
|
||||||
|
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||||
|
if (event.isFinal) {
|
||||||
|
if (this.transcript && !this.transcript.endsWith("\n")) {
|
||||||
|
this.transcript = `${this.transcript}\n`;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.transcript = appendTranscriptChunk(this.transcript, event.text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendTranscriptChunk(transcript: string, chunk: string): string {
|
||||||
|
const trimmed = chunk.replace(/\s+/g, " ").trim();
|
||||||
|
if (!trimmed) return transcript;
|
||||||
|
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
|
||||||
|
return `${transcript}${trimmed}`;
|
||||||
|
}
|
||||||
|
return `${transcript} ${trimmed}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function wrapTranscribeText(font: typeof dashboardFont, text: string, maxWidth: number): string[] {
|
||||||
|
const words = text.split(/\s+/).filter(Boolean);
|
||||||
|
const lines: string[] = [];
|
||||||
|
let line = "";
|
||||||
|
for (const word of words) {
|
||||||
|
const candidate = line ? `${line} ${word}` : word;
|
||||||
|
if (line && font.measureText(candidate) > maxWidth) {
|
||||||
|
lines.push(line);
|
||||||
|
line = word;
|
||||||
|
} else {
|
||||||
|
line = candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (line) {
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
return lines.length ? lines : [""];
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateTranscribeLine(text: string, maxChars: number): string {
|
||||||
|
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
|
||||||
|
}
|
||||||
|
|
||||||
function formatStopwatchElapsed(elapsedMs: number): string {
|
function formatStopwatchElapsed(elapsedMs: number): string {
|
||||||
const totalTenths = Math.floor(elapsedMs / 100);
|
const totalTenths = Math.floor(elapsedMs / 100);
|
||||||
const tenths = totalTenths % 10;
|
const tenths = totalTenths % 10;
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ export type LayerActions = {
|
|||||||
endTextSettingEdit: () => Promise<void> | void;
|
endTextSettingEdit: () => Promise<void> | void;
|
||||||
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
|
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
|
||||||
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
|
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
|
||||||
|
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
|
||||||
|
startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise<void> | void;
|
||||||
|
stopDedicatedVoiceInput: () => Promise<void> | void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type PaintBelow = () => GrayImage;
|
export type PaintBelow = () => GrayImage;
|
||||||
|
|||||||
Reference in new issue
Block a user