Merge main into ios-port and update the iOS CFW transport

This commit is contained in:
James Babcock committed 2026-09-17 23:33:51 -07:00
commit e8e144f3f2
156 files changed
+10528 -1679

No files matched your search

+1 -1
View File
@@ -1,6 +1,6 @@
/**
* Ambient light sensor access through the custom firmware (CFW image-handler
* mode 16, capability token "als16"; see g2flash/patches/als_sensor.c).
* mode 16; see g2flash/patches/als_sensor.c).
*
* The stock firmware only reads its OPT3001 light sensor while auto-brightness
* is on, and then steps the panel brightness itself (visible flicker). The CFW
+172 -95
View File
@@ -4,50 +4,107 @@ declare const com: any;
declare const java: any;
/**
* Download management for the on-device transcription model (Moonshine, via
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
* demand into the same filesDir location that earlier releases copied the
* bundled files to, so upgraded installs that already used on-device
* transcription need no download. FaceclawVoiceController reads the files
* from this directory.
* Download management for the on-device transcription models (sherpa-onnx
* offline recognizers). Models are fetched on demand into filesDir rather
* than bundled in the APK; FaceclawVoiceController reads the files from the
* directory each model's `dirName` names.
*
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
* three files rather than one; they download sequentially through
* Two models, same shape:
* - "moonshine": the original on-device option (three files). The model is
* no longer bundled in the APK; it is fetched into the same filesDir
* location that earlier releases copied the bundled files to, so
* upgraded installs that already used on-device transcription need no
* download.
* - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en,
* int8-quantized. Picked over tiny.en for materially better accuracy
* (Whisper's own tiny/base WER gap is real and well documented) on the
* reasoning that FaceclawVoiceController.java no longer re-decodes a
* Whisper segment on every live-partial tick — see its ASR_WHISPER_*
* comment — so base.en's extra compute is a one-time cost per utterance,
* not a cost paid ~1.4x/second while the user is speaking. tiny.en is an
* easy swap (this file's numbers only) if base.en proves too slow on a
* real phone; nobody has measured that yet. Files are csukuangfj's own
* (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx
* release asset, matching the mirror-not-the-GH-tarball approach already
* used for Moonshine below. Every hash here was computed locally from a
* freshly downloaded copy of the file, not copied from a doc page.
*
* Mirrors the on-phone assistant model flow in llama.ts, except each model
* here is multiple files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file).
*/
export type AsrModelId = "moonshine" | "whisper-base-en";
type AsrModelFile = {
name: string;
sha256: string;
sizeBytes: number;
};
export const ASR_MODEL = {
label: "Moonshine (English)",
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
// this mirror serves the same files (sha256-verified) individually.
baseUrl:
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
files: [
{
name: "decoder_model_merged.ort",
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
sizeBytes: 109424400,
},
{
name: "encoder_model.ort",
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
sizeBytes: 31326816,
},
{
name: "tokens.txt",
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
sizeBytes: 549350,
},
] as AsrModelFile[],
totalBytes: 141300566,
type AsrModelDef = {
label: string;
dirName: string;
baseUrl: string;
files: AsrModelFile[];
totalBytes: number;
};
export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
moonshine: {
label: "Moonshine (English)",
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
// this mirror serves the same files (sha256-verified) individually.
baseUrl:
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
files: [
{
name: "decoder_model_merged.ort",
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
sizeBytes: 109424400,
},
{
name: "encoder_model.ort",
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
sizeBytes: 31326816,
},
{
name: "tokens.txt",
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
sizeBytes: 549350,
},
],
totalBytes: 141300566,
},
"whisper-base-en": {
label: "Whisper (English, base)",
dirName: "sherpa-onnx-whisper-base-en-int8",
// csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx
// maintainer's own mirror of the project's whisper export, individual
// files (no tar.bz2-unpack problem to begin with, but kept as the same
// per-file-with-pinned-hash shape as Moonshine above for consistency).
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/",
files: [
{
name: "base.en-encoder.int8.onnx",
sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823",
sizeBytes: 29120534,
},
{
name: "base.en-decoder.int8.onnx",
sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41",
sizeBytes: 130669978,
},
{
name: "base.en-tokens.txt",
sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
sizeBytes: 835554,
},
],
totalBytes: 160626066,
},
};
export type AsrModelState = {
@@ -56,121 +113,141 @@ export type AsrModelState = {
totalBytes: number;
};
let downloader: any = null;
// Bytes of files already fully downloaded in this run, plus progress within
// the file currently downloading; drives the aggregate percentage.
let completedBytes = 0;
let currentFileBytes = 0;
const stateListeners = new Set<(state: AsrModelState) => void>();
type ModelRuntime = {
downloader: any;
// Bytes of files already fully downloaded in this run, plus progress within
// the file currently downloading; drives the aggregate percentage.
completedBytes: number;
currentFileBytes: number;
stateListeners: Set<(state: AsrModelState) => void>;
};
function modelDirPath(): string {
function freshRuntime(): ModelRuntime {
return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() };
}
const runtimes: Record<AsrModelId, ModelRuntime> = {
moonshine: freshRuntime(),
"whisper-base-en": freshRuntime(),
};
function modelDirPath(id: AsrModelId): string {
const context = Utils.android.getApplicationContext();
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`;
}
function filePath(file: AsrModelFile): string {
return `${modelDirPath()}/${file.name}`;
function filePath(id: AsrModelId, file: AsrModelFile): string {
return `${modelDirPath(id)}/${file.name}`;
}
function isFilePresent(file: AsrModelFile): boolean {
function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean {
try {
const javaFile = new java.io.File(filePath(file));
const javaFile = new java.io.File(filePath(id, file));
return javaFile.exists() && javaFile.length() > 0;
} catch {
return false;
}
}
export function isAsrModelReady(): boolean {
export function isAsrModelReady(id: AsrModelId): boolean {
if (!global.isAndroid) return false;
return ASR_MODEL.files.every(isFilePresent);
return ASR_MODELS[id].files.every((file) => isFilePresent(id, file));
}
export function asrModelState(): AsrModelState {
if (downloader) {
export function asrModelState(id: AsrModelId): AsrModelState {
const runtime = runtimes[id];
const totalBytes = ASR_MODELS[id].totalBytes;
if (runtime.downloader) {
return {
status: "downloading",
bytesDownloaded: completedBytes + currentFileBytes,
totalBytes: ASR_MODEL.totalBytes,
bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes,
totalBytes,
};
}
return {
status: isAsrModelReady() ? "ready" : "absent",
status: isAsrModelReady(id) ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: ASR_MODEL.totalBytes,
totalBytes,
};
}
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void {
const runtime = runtimes[id];
runtime.stateListeners.add(listener);
return () => runtime.stateListeners.delete(listener);
}
function notifyStateChanged(): void {
const state = asrModelState();
stateListeners.forEach((listener) => listener(state));
function notifyStateChanged(id: AsrModelId): void {
const state = asrModelState(id);
runtimes[id].stateListeners.forEach((listener) => listener(state));
}
export function startAsrModelDownload(): void {
if (!global.isAndroid || downloader || isAsrModelReady()) return;
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
downloadNextFile();
notifyStateChanged();
export function startAsrModelDownload(id: AsrModelId): void {
const runtime = runtimes[id];
if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return;
runtime.completedBytes = ASR_MODELS[id].files
.filter((file) => isFilePresent(id, file))
.reduce((sum, f) => sum + f.sizeBytes, 0);
downloadNextFile(id);
notifyStateChanged(id);
}
function downloadNextFile(): void {
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
function downloadNextFile(id: AsrModelId): void {
const runtime = runtimes[id];
const def = ASR_MODELS[id];
const nextFile = def.files.find((file) => !isFilePresent(id, file));
if (!nextFile) {
downloader = null;
notifyStateChanged();
runtime.downloader = null;
notifyStateChanged(id);
return;
}
currentFileBytes = 0;
runtime.currentFileBytes = 0;
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
currentFileBytes = Number(bytes);
notifyStateChanged();
runtime.currentFileBytes = Number(bytes);
notifyStateChanged(id);
},
onDone: () => {
completedBytes += nextFile.sizeBytes;
currentFileBytes = 0;
downloadNextFile();
runtime.completedBytes += nextFile.sizeBytes;
runtime.currentFileBytes = 0;
downloadNextFile(id);
},
onError: (message: string) => {
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
downloader = null;
notifyStateChanged();
console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`);
runtime.downloader = null;
notifyStateChanged(id);
},
});
downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${ASR_MODEL.baseUrl}${nextFile.name}`,
filePath(nextFile),
runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${def.baseUrl}${nextFile.name}`,
filePath(id, nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloader.start();
runtime.downloader.start();
}
/** Stops the download; already-fetched bytes are kept and resumed next time. */
export function cancelAsrModelDownload(): void {
if (!downloader) return;
downloader.cancel();
downloader = null;
notifyStateChanged();
export function cancelAsrModelDownload(id: AsrModelId): void {
const runtime = runtimes[id];
if (!runtime.downloader) return;
runtime.downloader.cancel();
runtime.downloader = null;
notifyStateChanged(id);
}
export function deleteAsrModel(): void {
export function deleteAsrModel(id: AsrModelId): void {
if (!global.isAndroid) return;
cancelAsrModelDownload();
cancelAsrModelDownload(id);
try {
for (const file of ASR_MODEL.files) {
new java.io.File(filePath(file)).delete();
new java.io.File(`${filePath(file)}.part`).delete();
for (const file of ASR_MODELS[id].files) {
new java.io.File(filePath(id, file)).delete();
new java.io.File(`${filePath(id, file)}.part`).delete();
}
new java.io.File(modelDirPath()).delete();
new java.io.File(modelDirPath(id)).delete();
} catch (error) {
console.error(`Voice model delete failed: ${String(error)}`);
console.error(`Voice model delete failed (${id}): ${String(error)}`);
}
notifyStateChanged();
notifyStateChanged(id);
}
+71
View File
@@ -0,0 +1,71 @@
declare const com: any;
/**
* BLE bandwidth benchmark controls (Developer app). The Java communicator
* streams no-op image payloads for a fixed duration with a selectable message
* size and pipeline window, then reports throughput. Progress and results are
* polled with getBandwidthBenchmarkStatus.
*
* Talks to the active communicator via the FaceclawBleCommunicator.getActive()
* static, so it works from any isolate without threading the bridge through.
*/
export type BandwidthBenchmarkStatus = {
state: "idle" | "starting" | "running" | "done";
messageSize: number;
windowSize: number;
linkMode: number;
elapsedMs: number;
messagesSent: number;
messagesAcked: number;
timeouts: number;
payloadBytesAcked: number;
wireBytesAcked: number;
aborted: boolean;
};
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Start a run. False if not connected or a run is already active. */
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
const active = activeCommunicator();
if (!active) return false;
try {
return Boolean(
active.startBandwidthBenchmarkWithLinkMode(
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode),
),
);
} catch (error) {
console.warn(`startBandwidthBenchmark failed: ${error}`);
return false;
}
}
/** Cancel an in-progress run. Safe to call when idle. */
export function cancelBandwidthBenchmark(): void {
try {
activeCommunicator()?.cancelBandwidthBenchmark();
} catch (error) {
console.warn(`cancelBandwidthBenchmark failed: ${error}`);
}
}
/** Status of the current or most recent run; null when not connected. */
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
const active = activeCommunicator();
if (!active) return null;
try {
return JSON.parse(String(active.getBandwidthBenchmarkStatus())) as BandwidthBenchmarkStatus;
} catch (error) {
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`);
return null;
}
}
+6 -8
View File
@@ -1,19 +1,17 @@
import { Utils } from "@nativescript/core";
import { type FirmwareInfo } from "../g2/firmware-compat";
declare const com: any;
export type DeviceInfo = {
leftVersion: string;
rightVersion: string;
capabilities: string;
};
export type DeviceInfo = FirmwareInfo;
export type DeviceInfoState = "connecting" | "authenticating" | "querying";
/**
* TS wrapper around the native FaceclawDeviceInfoProbe — a one-shot,
* stock-compatible connect + firmware/device-info read. `run()` resolves with
* the firmware versions and CFW capability string, or rejects on failure.
* the firmware versions and the firmware-extension string, or rejects on failure.
*/
export class DeviceInfoProbe {
private readonly probe: any;
@@ -34,11 +32,11 @@ export class DeviceInfoProbe {
onLog: (line: string) => this.emit(this.logListeners, String(line)),
onState: (state: string, detail: string) =>
this.emit(this.stateListeners, String(state) as DeviceInfoState, String(detail ?? "")),
onResult: (leftVersion: string, rightVersion: string, capabilities: string) =>
onResult: (leftVersion: string, rightVersion: string, extension: string) =>
this.settle(null, {
leftVersion: String(leftVersion),
rightVersion: String(rightVersion),
capabilities: String(capabilities),
extension: String(extension),
}),
onError: (message: string) => this.settle(new Error(String(message)), null),
});
+14 -6
View File
@@ -1,4 +1,4 @@
import { Utils } from "@nativescript/core";
import { Dialogs, Utils } from "@nativescript/core";
declare const com: any;
@@ -23,11 +23,19 @@ export function readEvenAppNotificationState(): EvenAppNotificationState {
}
export function openEvenAppSettings(): void {
if (!global.isAndroid) return;
const context = Utils.android.getApplicationContext();
if (context) {
com.faceclaw.app.FaceclawEvenAppDetector.openEvenAppSettings(context);
}
const context = global.isAndroid ? Utils.android.getApplicationContext() : null;
const result = context
? com.faceclaw.app.FaceclawEvenAppDetector.openEvenAppSettings(context)
: "failed";
if (result === "opened") return;
const notInstalled = result === "not-installed";
void Dialogs.alert({
title: notInstalled ? "Even app not installed" : "Unable to open Even app settings",
message: notInstalled
? "The official Even Realities app isn't installed on this phone. If your glasses are connected to another app or phone, disconnect them there. Otherwise, you can continue."
: "Open your phone's settings manually to manage the Even app, or disconnect your glasses from within the app that's connected to them.",
okButtonText: "OK",
});
}
export function openNotificationAccessSettings(): void {
+11 -9
View File
@@ -22,6 +22,8 @@ export type CommunicatorState = {
export type HeadsetBatteryState = {
battery: number;
chargingStatus: number;
ringBattery?: number;
ringChargingStatus?: number;
};
export type FrameMetrics = {
@@ -30,11 +32,9 @@ export type FrameMetrics = {
tileCount: number;
};
export type FirmwareInfo = {
leftVersion: string;
rightVersion: string;
capabilities: string;
};
import { type FirmwareInfo } from "../g2/firmware-compat";
export type { FirmwareInfo };
/**
* Compositor surface configuration. Position/size are in screen pixels;
@@ -189,10 +189,12 @@ export class FaceclawCommunicatorBridge {
frameTimings.logFrame(event.frameId, "input event received on JS side");
this.emitAsync(this.ringListeners, event);
},
onBatteryState: (headsetBattery: number, headsetCharging: number) => {
onBatteryState: (headsetBattery: number, headsetCharging: number, ringBattery: number, ringCharging: number) => {
const state = {
battery: Number(headsetBattery),
chargingStatus: Number(headsetCharging),
ringBattery: Number(ringBattery),
ringChargingStatus: Number(ringCharging),
};
this.emitAsync(this.batteryListeners, state);
},
@@ -226,11 +228,11 @@ export class FaceclawCommunicatorBridge {
onFrameFinished: (frameId: number, outcome: string) => {
this.recordFrameFinished(Number(frameId), String(outcome));
},
onFirmwareInfo: (leftVersion: string, rightVersion: string, capabilities: string) => {
const info = {
onFirmwareInfo: (leftVersion: string, rightVersion: string, extension: string) => {
const info: FirmwareInfo = {
leftVersion: String(leftVersion),
rightVersion: String(rightVersion),
capabilities: String(capabilities),
extension: String(extension),
};
this.emitAsync(this.firmwareInfoListeners, info);
},
+25
View File
@@ -0,0 +1,25 @@
/**
* Open a URL in the phone's browser. Uses the application context with
* NEW_TASK so it works from worker threads (no Activity in hand). Returns
* false when no browser is available or on a non-Android platform.
*/
import { Utils } from "@nativescript/core";
declare const android: any;
declare const global: any;
export function openUrlOnPhone(url: string): boolean {
if (!global.isAndroid) return false;
try {
const context = Utils.android.getApplicationContext();
if (!context) return false;
const intent = new android.content.Intent(android.content.Intent.ACTION_VIEW, android.net.Uri.parse(url));
intent.addCategory(android.content.Intent.CATEGORY_BROWSABLE);
intent.addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK);
context.startActivity(intent);
return true;
} catch (error) {
console.warn(`open-url: could not open ${url}: ${error}`);
return false;
}
}
+2 -2
View File
@@ -23,14 +23,14 @@ export class IosVoiceControlBridge {
private log: (message: string) => void = () => {}
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status }); return () => { this.statuses.delete(listener) }
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
}
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
this.transcripts.add(listener); return () => { this.transcripts.delete(listener) }
}
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status }) }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
private ensureNative(): any {
if (!this.native) {
this.native = FaceclawSpeech.new()
+77 -7
View File
@@ -11,10 +11,27 @@ import { toUint8Array } from "../util/array-util";
declare const com: any;
export type VoiceControlState = {
/** One-line state of the capture: progress, "Listening...", or an error. */
status: string;
/**
* Whether audio is actually being captured and transcribed right now. False
* while the mic is starting, while the phone is showing the permission
* prompt, and after an error, so a dialog must not claim to be listening.
*/
listening: boolean;
/**
* Longer guidance to show in place of the (still empty) transcript, e.g.
* what the user has to do on the phone. Empty when there is nothing to say.
*/
detail: string;
};
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
// renamed, since it's a persisted settings value on real installs already.
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
@@ -88,7 +105,12 @@ export class FaceclawVoiceControlBridge {
private controller: any | null = null;
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
private listening = false;
private detail = "";
private started = false;
private nativeCaptureFinished: Promise<void> = Promise.resolve();
private nextNativeCaptureId = 1;
private readonly nativeCaptureResolvers = new Map<number, () => void>();
// The mic is a single shared stream; these are the reasons it is running.
// The first holder starts capture (choosing the provider); the mic stops
// when the last one releases. Transcript events broadcast to every
@@ -118,7 +140,7 @@ export class FaceclawVoiceControlBridge {
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
listener({ status: this.status });
listener(this.state());
return () => this.statusListeners.delete(listener);
}
@@ -142,14 +164,41 @@ export class FaceclawVoiceControlBridge {
return () => this.speechPauseListeners.delete(listener);
}
/**
* The capture is blocked on the phone's system microphone-permission
* prompt. Reported by the controller before it asks Android, so the dialog
* on the glasses can send the user to the phone instead of claiming to
* listen; the next status (from the capture start, or reportPermissionDenied)
* replaces it.
*/
reportPermissionPrompt(): void {
this.setStatus(
"Waiting for microphone permission...",
"Allow microphone access in the prompt on your phone.",
);
}
/** The user declined the microphone permission (or the prompt could not be shown). */
reportPermissionDenied(): void {
this.setStatus(
"Microphone access denied.",
"Enable the microphone for Faceclaw in the phone's app settings, then try again.",
);
}
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
}
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
stopPushToTalk(): void {
stopPushToTalk(): Promise<void> | void {
// Shared continuous capture keeps running; cloud providers finalize via
// their own callbacks. Only a stopped native recognizer must be awaited.
const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1
? this.nativeCaptureFinished : undefined;
this.releaseCapture("ptt", true);
return finished;
}
/** Begin continuous capture (Transcribe): the mic stays on until released. */
@@ -262,6 +311,9 @@ export class FaceclawVoiceControlBridge {
}
this.speechPause.reset();
// Replaces whatever the last capture left behind ("Listening...", an
// error, the permission prompt) until the controller reports progress.
this.setStatus("Starting microphone...");
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
@@ -275,7 +327,12 @@ export class FaceclawVoiceControlBridge {
this.cloudClient = null;
this.started = true;
this.controller?.start("onboard");
// Which on-device model to load; a no-op setter for every provider except
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
const captureId = this.nextNativeCaptureId++;
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
this.controller?.start("onboard", captureId);
}
/**
@@ -284,7 +341,7 @@ export class FaceclawVoiceControlBridge {
* than failing the capture outright.
*/
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
if (options.provider === "onboard") return null;
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
const sttOptions = {
apiKey: "",
onTranscript: (event: CloudSttTranscriptEvent) =>
@@ -444,6 +501,10 @@ export class FaceclawVoiceControlBridge {
onTranscript: (text: string, isFinal: boolean) => {
this.emitTranscript(String(text), Boolean(isFinal));
},
onStopped: (captureId: number) => {
this.nativeCaptureResolvers.get(captureId)?.();
this.nativeCaptureResolvers.delete(captureId);
},
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
@@ -492,12 +553,21 @@ export class FaceclawVoiceControlBridge {
}
}
private setStatus(status: string): void {
private setStatus(status: string, detail = ""): void {
this.status = status;
this.detail = detail;
// Every "audio is flowing" report — from the Java controller and from the
// cloud providers — is phrased "Listening..." / "Listening (X)...".
this.listening = status.startsWith("Listening");
const state = this.state();
for (const listener of this.statusListeners) {
listener({ status });
listener(state);
}
}
private state(): VoiceControlState {
return { status: this.status, listening: this.listening, detail: this.detail };
}
}
function toJavaFloats(values: Float32Array): any {
+10 -1
View File
@@ -24,6 +24,10 @@ export const WEAR_PATHS = {
text: "/faceclaw/text",
/** Watch -> phone: please re-publish the state item. */
stateRequest: "/faceclaw/state/request",
/** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */
battery: "/faceclaw/battery",
/** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */
batteryRequest: "/faceclaw/battery/request",
/** Phone -> watch: reply to any of the above. */
ack: "/faceclaw/ack",
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
@@ -157,7 +161,12 @@ class WearBridge {
/** Send an event to every reachable watch. */
sendEvent(payload: Record<string, unknown>): void {
this.getJava()?.sendToWatch(WEAR_PATHS.event, JSON.stringify(payload));
this.sendToWatch(WEAR_PATHS.event, payload);
}
/** Send a message on any phone -> watch path to every reachable watch. */
sendToWatch(path: string, payload: Record<string, unknown>): void {
this.getJava()?.sendToWatch(path, JSON.stringify(payload));
}
/** Acknowledge a watch message; `seq` echoes the watch's sequence number. */