forked from Blendman974/faceclaw
Merge main into ios-port and update the iOS CFW transport
This commit is contained in:
commit
e8e144f3f2
156 files changed
+10528
-1679
No files matched your search
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Ambient light sensor access through the custom firmware (CFW image-handler
|
||||
* mode 16, capability token "als16"; see g2flash/patches/als_sensor.c).
|
||||
* mode 16; see g2flash/patches/als_sensor.c).
|
||||
*
|
||||
* The stock firmware only reads its OPT3001 light sensor while auto-brightness
|
||||
* is on, and then steps the panel brightness itself (visible flicker). The CFW
|
||||
|
||||
+172
-95
@@ -4,50 +4,107 @@ declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Download management for the on-device transcription model (Moonshine, via
|
||||
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
|
||||
* demand into the same filesDir location that earlier releases copied the
|
||||
* bundled files to, so upgraded installs that already used on-device
|
||||
* transcription need no download. FaceclawVoiceController reads the files
|
||||
* from this directory.
|
||||
* Download management for the on-device transcription models (sherpa-onnx
|
||||
* offline recognizers). Models are fetched on demand into filesDir rather
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
|
||||
* three files rather than one; they download sequentially through
|
||||
* Two models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
* upgraded installs that already used on-device transcription need no
|
||||
* download.
|
||||
* - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en,
|
||||
* int8-quantized. Picked over tiny.en for materially better accuracy
|
||||
* (Whisper's own tiny/base WER gap is real and well documented) on the
|
||||
* reasoning that FaceclawVoiceController.java no longer re-decodes a
|
||||
* Whisper segment on every live-partial tick — see its ASR_WHISPER_*
|
||||
* comment — so base.en's extra compute is a one-time cost per utterance,
|
||||
* not a cost paid ~1.4x/second while the user is speaking. tiny.en is an
|
||||
* easy swap (this file's numbers only) if base.en proves too slow on a
|
||||
* real phone; nobody has measured that yet. Files are csukuangfj's own
|
||||
* (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx
|
||||
* release asset, matching the mirror-not-the-GH-tarball approach already
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODEL = {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
] as AsrModelFile[],
|
||||
totalBytes: 141300566,
|
||||
type AsrModelDef = {
|
||||
label: string;
|
||||
dirName: string;
|
||||
baseUrl: string;
|
||||
files: AsrModelFile[];
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
moonshine: {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
],
|
||||
totalBytes: 141300566,
|
||||
},
|
||||
"whisper-base-en": {
|
||||
label: "Whisper (English, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-en-int8",
|
||||
// csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx
|
||||
// maintainer's own mirror of the project's whisper export, individual
|
||||
// files (no tar.bz2-unpack problem to begin with, but kept as the same
|
||||
// per-file-with-pinned-hash shape as Moonshine above for consistency).
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "base.en-encoder.int8.onnx",
|
||||
sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base.en-decoder.int8.onnx",
|
||||
sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41",
|
||||
sizeBytes: 130669978,
|
||||
},
|
||||
{
|
||||
name: "base.en-tokens.txt",
|
||||
sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
||||
sizeBytes: 835554,
|
||||
},
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -56,121 +113,141 @@ export type AsrModelState = {
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let downloader: any = null;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
let completedBytes = 0;
|
||||
let currentFileBytes = 0;
|
||||
const stateListeners = new Set<(state: AsrModelState) => void>();
|
||||
type ModelRuntime = {
|
||||
downloader: any;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
completedBytes: number;
|
||||
currentFileBytes: number;
|
||||
stateListeners: Set<(state: AsrModelState) => void>;
|
||||
};
|
||||
|
||||
function modelDirPath(): string {
|
||||
function freshRuntime(): ModelRuntime {
|
||||
return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() };
|
||||
}
|
||||
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`;
|
||||
}
|
||||
|
||||
function filePath(file: AsrModelFile): string {
|
||||
return `${modelDirPath()}/${file.name}`;
|
||||
function filePath(id: AsrModelId, file: AsrModelFile): string {
|
||||
return `${modelDirPath(id)}/${file.name}`;
|
||||
}
|
||||
|
||||
function isFilePresent(file: AsrModelFile): boolean {
|
||||
function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean {
|
||||
try {
|
||||
const javaFile = new java.io.File(filePath(file));
|
||||
const javaFile = new java.io.File(filePath(id, file));
|
||||
return javaFile.exists() && javaFile.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAsrModelReady(): boolean {
|
||||
export function isAsrModelReady(id: AsrModelId): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
return ASR_MODEL.files.every(isFilePresent);
|
||||
return ASR_MODELS[id].files.every((file) => isFilePresent(id, file));
|
||||
}
|
||||
|
||||
export function asrModelState(): AsrModelState {
|
||||
if (downloader) {
|
||||
export function asrModelState(id: AsrModelId): AsrModelState {
|
||||
const runtime = runtimes[id];
|
||||
const totalBytes = ASR_MODELS[id].totalBytes;
|
||||
if (runtime.downloader) {
|
||||
return {
|
||||
status: "downloading",
|
||||
bytesDownloaded: completedBytes + currentFileBytes,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: isAsrModelReady() ? "ready" : "absent",
|
||||
status: isAsrModelReady(id) ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void {
|
||||
const runtime = runtimes[id];
|
||||
runtime.stateListeners.add(listener);
|
||||
return () => runtime.stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = asrModelState();
|
||||
stateListeners.forEach((listener) => listener(state));
|
||||
function notifyStateChanged(id: AsrModelId): void {
|
||||
const state = asrModelState(id);
|
||||
runtimes[id].stateListeners.forEach((listener) => listener(state));
|
||||
}
|
||||
|
||||
export function startAsrModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isAsrModelReady()) return;
|
||||
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile();
|
||||
notifyStateChanged();
|
||||
export function startAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return;
|
||||
runtime.completedBytes = ASR_MODELS[id].files
|
||||
.filter((file) => isFilePresent(id, file))
|
||||
.reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
function downloadNextFile(): void {
|
||||
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
|
||||
function downloadNextFile(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
const def = ASR_MODELS[id];
|
||||
const nextFile = def.files.find((file) => !isFilePresent(id, file));
|
||||
if (!nextFile) {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
return;
|
||||
}
|
||||
currentFileBytes = 0;
|
||||
runtime.currentFileBytes = 0;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
currentFileBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
runtime.currentFileBytes = Number(bytes);
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
onDone: () => {
|
||||
completedBytes += nextFile.sizeBytes;
|
||||
currentFileBytes = 0;
|
||||
downloadNextFile();
|
||||
runtime.completedBytes += nextFile.sizeBytes;
|
||||
runtime.currentFileBytes = 0;
|
||||
downloadNextFile(id);
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`);
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${ASR_MODEL.baseUrl}${nextFile.name}`,
|
||||
filePath(nextFile),
|
||||
runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${def.baseUrl}${nextFile.name}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloader.start();
|
||||
runtime.downloader.start();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelAsrModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
export function cancelAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!runtime.downloader) return;
|
||||
runtime.downloader.cancel();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
export function deleteAsrModel(): void {
|
||||
export function deleteAsrModel(id: AsrModelId): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelAsrModelDownload();
|
||||
cancelAsrModelDownload(id);
|
||||
try {
|
||||
for (const file of ASR_MODEL.files) {
|
||||
new java.io.File(filePath(file)).delete();
|
||||
new java.io.File(`${filePath(file)}.part`).delete();
|
||||
for (const file of ASR_MODELS[id].files) {
|
||||
new java.io.File(filePath(id, file)).delete();
|
||||
new java.io.File(`${filePath(id, file)}.part`).delete();
|
||||
}
|
||||
new java.io.File(modelDirPath()).delete();
|
||||
new java.io.File(modelDirPath(id)).delete();
|
||||
} catch (error) {
|
||||
console.error(`Voice model delete failed: ${String(error)}`);
|
||||
console.error(`Voice model delete failed (${id}): ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* BLE bandwidth benchmark controls (Developer app). The Java communicator
|
||||
* streams no-op image payloads for a fixed duration with a selectable message
|
||||
* size and pipeline window, then reports throughput. Progress and results are
|
||||
* polled with getBandwidthBenchmarkStatus.
|
||||
*
|
||||
* Talks to the active communicator via the FaceclawBleCommunicator.getActive()
|
||||
* static, so it works from any isolate without threading the bridge through.
|
||||
*/
|
||||
|
||||
export type BandwidthBenchmarkStatus = {
|
||||
state: "idle" | "starting" | "running" | "done";
|
||||
messageSize: number;
|
||||
windowSize: number;
|
||||
linkMode: number;
|
||||
elapsedMs: number;
|
||||
messagesSent: number;
|
||||
messagesAcked: number;
|
||||
timeouts: number;
|
||||
payloadBytesAcked: number;
|
||||
wireBytesAcked: number;
|
||||
aborted: boolean;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Start a run. False if not connected or a run is already active. */
|
||||
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return false;
|
||||
try {
|
||||
return Boolean(
|
||||
active.startBandwidthBenchmarkWithLinkMode(
|
||||
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`startBandwidthBenchmark failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Cancel an in-progress run. Safe to call when idle. */
|
||||
export function cancelBandwidthBenchmark(): void {
|
||||
try {
|
||||
activeCommunicator()?.cancelBandwidthBenchmark();
|
||||
} catch (error) {
|
||||
console.warn(`cancelBandwidthBenchmark failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Status of the current or most recent run; null when not connected. */
|
||||
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return null;
|
||||
try {
|
||||
return JSON.parse(String(active.getBandwidthBenchmarkStatus())) as BandwidthBenchmarkStatus;
|
||||
} catch (error) {
|
||||
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,17 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type FirmwareInfo } from "../g2/firmware-compat";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type DeviceInfo = {
|
||||
leftVersion: string;
|
||||
rightVersion: string;
|
||||
capabilities: string;
|
||||
};
|
||||
export type DeviceInfo = FirmwareInfo;
|
||||
|
||||
export type DeviceInfoState = "connecting" | "authenticating" | "querying";
|
||||
|
||||
/**
|
||||
* TS wrapper around the native FaceclawDeviceInfoProbe — a one-shot,
|
||||
* stock-compatible connect + firmware/device-info read. `run()` resolves with
|
||||
* the firmware versions and CFW capability string, or rejects on failure.
|
||||
* the firmware versions and the firmware-extension string, or rejects on failure.
|
||||
*/
|
||||
export class DeviceInfoProbe {
|
||||
private readonly probe: any;
|
||||
@@ -34,11 +32,11 @@ export class DeviceInfoProbe {
|
||||
onLog: (line: string) => this.emit(this.logListeners, String(line)),
|
||||
onState: (state: string, detail: string) =>
|
||||
this.emit(this.stateListeners, String(state) as DeviceInfoState, String(detail ?? "")),
|
||||
onResult: (leftVersion: string, rightVersion: string, capabilities: string) =>
|
||||
onResult: (leftVersion: string, rightVersion: string, extension: string) =>
|
||||
this.settle(null, {
|
||||
leftVersion: String(leftVersion),
|
||||
rightVersion: String(rightVersion),
|
||||
capabilities: String(capabilities),
|
||||
extension: String(extension),
|
||||
}),
|
||||
onError: (message: string) => this.settle(new Error(String(message)), null),
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { Dialogs, Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -23,11 +23,19 @@ export function readEvenAppNotificationState(): EvenAppNotificationState {
|
||||
}
|
||||
|
||||
export function openEvenAppSettings(): void {
|
||||
if (!global.isAndroid) return;
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (context) {
|
||||
com.faceclaw.app.FaceclawEvenAppDetector.openEvenAppSettings(context);
|
||||
}
|
||||
const context = global.isAndroid ? Utils.android.getApplicationContext() : null;
|
||||
const result = context
|
||||
? com.faceclaw.app.FaceclawEvenAppDetector.openEvenAppSettings(context)
|
||||
: "failed";
|
||||
if (result === "opened") return;
|
||||
const notInstalled = result === "not-installed";
|
||||
void Dialogs.alert({
|
||||
title: notInstalled ? "Even app not installed" : "Unable to open Even app settings",
|
||||
message: notInstalled
|
||||
? "The official Even Realities app isn't installed on this phone. If your glasses are connected to another app or phone, disconnect them there. Otherwise, you can continue."
|
||||
: "Open your phone's settings manually to manage the Even app, or disconnect your glasses from within the app that's connected to them.",
|
||||
okButtonText: "OK",
|
||||
});
|
||||
}
|
||||
|
||||
export function openNotificationAccessSettings(): void {
|
||||
|
||||
@@ -22,6 +22,8 @@ export type CommunicatorState = {
|
||||
export type HeadsetBatteryState = {
|
||||
battery: number;
|
||||
chargingStatus: number;
|
||||
ringBattery?: number;
|
||||
ringChargingStatus?: number;
|
||||
};
|
||||
|
||||
export type FrameMetrics = {
|
||||
@@ -30,11 +32,9 @@ export type FrameMetrics = {
|
||||
tileCount: number;
|
||||
};
|
||||
|
||||
export type FirmwareInfo = {
|
||||
leftVersion: string;
|
||||
rightVersion: string;
|
||||
capabilities: string;
|
||||
};
|
||||
import { type FirmwareInfo } from "../g2/firmware-compat";
|
||||
|
||||
export type { FirmwareInfo };
|
||||
|
||||
/**
|
||||
* Compositor surface configuration. Position/size are in screen pixels;
|
||||
@@ -189,10 +189,12 @@ export class FaceclawCommunicatorBridge {
|
||||
frameTimings.logFrame(event.frameId, "input event received on JS side");
|
||||
this.emitAsync(this.ringListeners, event);
|
||||
},
|
||||
onBatteryState: (headsetBattery: number, headsetCharging: number) => {
|
||||
onBatteryState: (headsetBattery: number, headsetCharging: number, ringBattery: number, ringCharging: number) => {
|
||||
const state = {
|
||||
battery: Number(headsetBattery),
|
||||
chargingStatus: Number(headsetCharging),
|
||||
ringBattery: Number(ringBattery),
|
||||
ringChargingStatus: Number(ringCharging),
|
||||
};
|
||||
this.emitAsync(this.batteryListeners, state);
|
||||
},
|
||||
@@ -226,11 +228,11 @@ export class FaceclawCommunicatorBridge {
|
||||
onFrameFinished: (frameId: number, outcome: string) => {
|
||||
this.recordFrameFinished(Number(frameId), String(outcome));
|
||||
},
|
||||
onFirmwareInfo: (leftVersion: string, rightVersion: string, capabilities: string) => {
|
||||
const info = {
|
||||
onFirmwareInfo: (leftVersion: string, rightVersion: string, extension: string) => {
|
||||
const info: FirmwareInfo = {
|
||||
leftVersion: String(leftVersion),
|
||||
rightVersion: String(rightVersion),
|
||||
capabilities: String(capabilities),
|
||||
extension: String(extension),
|
||||
};
|
||||
this.emitAsync(this.firmwareInfoListeners, info);
|
||||
},
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Open a URL in the phone's browser. Uses the application context with
|
||||
* NEW_TASK so it works from worker threads (no Activity in hand). Returns
|
||||
* false when no browser is available or on a non-Android platform.
|
||||
*/
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const android: any;
|
||||
declare const global: any;
|
||||
|
||||
export function openUrlOnPhone(url: string): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
try {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (!context) return false;
|
||||
const intent = new android.content.Intent(android.content.Intent.ACTION_VIEW, android.net.Uri.parse(url));
|
||||
intent.addCategory(android.content.Intent.CATEGORY_BROWSABLE);
|
||||
intent.addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK);
|
||||
context.startActivity(intent);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(`open-url: could not open ${url}: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -23,14 +23,14 @@ export class IosVoiceControlBridge {
|
||||
private log: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status }); return () => { this.statuses.delete(listener) }
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
}
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcripts.add(listener); return () => { this.transcripts.delete(listener) }
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status }) }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
|
||||
@@ -11,10 +11,27 @@ import { toUint8Array } from "../util/array-util";
|
||||
declare const com: any;
|
||||
|
||||
export type VoiceControlState = {
|
||||
/** One-line state of the capture: progress, "Listening...", or an error. */
|
||||
status: string;
|
||||
/**
|
||||
* Whether audio is actually being captured and transcribed right now. False
|
||||
* while the mic is starting, while the phone is showing the permission
|
||||
* prompt, and after an error, so a dialog must not claim to be listening.
|
||||
*/
|
||||
listening: boolean;
|
||||
/**
|
||||
* Longer guidance to show in place of the (still empty) transcript, e.g.
|
||||
* what the user has to do on the phone. Empty when there is nothing to say.
|
||||
*/
|
||||
detail: string;
|
||||
};
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -88,7 +105,12 @@ export class FaceclawVoiceControlBridge {
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private listening = false;
|
||||
private detail = "";
|
||||
private started = false;
|
||||
private nativeCaptureFinished: Promise<void> = Promise.resolve();
|
||||
private nextNativeCaptureId = 1;
|
||||
private readonly nativeCaptureResolvers = new Map<number, () => void>();
|
||||
// The mic is a single shared stream; these are the reasons it is running.
|
||||
// The first holder starts capture (choosing the provider); the mic stops
|
||||
// when the last one releases. Transcript events broadcast to every
|
||||
@@ -118,7 +140,7 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
listener({ status: this.status });
|
||||
listener(this.state());
|
||||
return () => this.statusListeners.delete(listener);
|
||||
}
|
||||
|
||||
@@ -142,14 +164,41 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.speechPauseListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* The capture is blocked on the phone's system microphone-permission
|
||||
* prompt. Reported by the controller before it asks Android, so the dialog
|
||||
* on the glasses can send the user to the phone instead of claiming to
|
||||
* listen; the next status (from the capture start, or reportPermissionDenied)
|
||||
* replaces it.
|
||||
*/
|
||||
reportPermissionPrompt(): void {
|
||||
this.setStatus(
|
||||
"Waiting for microphone permission...",
|
||||
"Allow microphone access in the prompt on your phone.",
|
||||
);
|
||||
}
|
||||
|
||||
/** The user declined the microphone permission (or the prompt could not be shown). */
|
||||
reportPermissionDenied(): void {
|
||||
this.setStatus(
|
||||
"Microphone access denied.",
|
||||
"Enable the microphone for Faceclaw in the phone's app settings, then try again.",
|
||||
);
|
||||
}
|
||||
|
||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("ptt", options);
|
||||
}
|
||||
|
||||
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
|
||||
stopPushToTalk(): void {
|
||||
stopPushToTalk(): Promise<void> | void {
|
||||
// Shared continuous capture keeps running; cloud providers finalize via
|
||||
// their own callbacks. Only a stopped native recognizer must be awaited.
|
||||
const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1
|
||||
? this.nativeCaptureFinished : undefined;
|
||||
this.releaseCapture("ptt", true);
|
||||
return finished;
|
||||
}
|
||||
|
||||
/** Begin continuous capture (Transcribe): the mic stays on until released. */
|
||||
@@ -262,6 +311,9 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
this.speechPause.reset();
|
||||
// Replaces whatever the last capture left behind ("Listening...", an
|
||||
// error, the permission prompt) until the controller reports progress.
|
||||
this.setStatus("Starting microphone...");
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
@@ -275,7 +327,12 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
this.controller?.start("onboard");
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -284,7 +341,7 @@ export class FaceclawVoiceControlBridge {
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard") return null;
|
||||
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
@@ -444,6 +501,10 @@ export class FaceclawVoiceControlBridge {
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onStopped: (captureId: number) => {
|
||||
this.nativeCaptureResolvers.get(captureId)?.();
|
||||
this.nativeCaptureResolvers.delete(captureId);
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
@@ -492,12 +553,21 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
private setStatus(status: string): void {
|
||||
private setStatus(status: string, detail = ""): void {
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
// Every "audio is flowing" report — from the Java controller and from the
|
||||
// cloud providers — is phrased "Listening..." / "Listening (X)...".
|
||||
this.listening = status.startsWith("Listening");
|
||||
const state = this.state();
|
||||
for (const listener of this.statusListeners) {
|
||||
listener({ status });
|
||||
listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
private state(): VoiceControlState {
|
||||
return { status: this.status, listening: this.listening, detail: this.detail };
|
||||
}
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
|
||||
@@ -24,6 +24,10 @@ export const WEAR_PATHS = {
|
||||
text: "/faceclaw/text",
|
||||
/** Watch -> phone: please re-publish the state item. */
|
||||
stateRequest: "/faceclaw/state/request",
|
||||
/** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */
|
||||
battery: "/faceclaw/battery",
|
||||
/** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */
|
||||
batteryRequest: "/faceclaw/battery/request",
|
||||
/** Phone -> watch: reply to any of the above. */
|
||||
ack: "/faceclaw/ack",
|
||||
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
|
||||
@@ -157,7 +161,12 @@ class WearBridge {
|
||||
|
||||
/** Send an event to every reachable watch. */
|
||||
sendEvent(payload: Record<string, unknown>): void {
|
||||
this.getJava()?.sendToWatch(WEAR_PATHS.event, JSON.stringify(payload));
|
||||
this.sendToWatch(WEAR_PATHS.event, payload);
|
||||
}
|
||||
|
||||
/** Send a message on any phone -> watch path to every reachable watch. */
|
||||
sendToWatch(path: string, payload: Record<string, unknown>): void {
|
||||
this.getJava()?.sendToWatch(path, JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/** Acknowledge a watch message; `seq` echoes the watch's sequence number. */
|
||||
|
||||
Reference in new issue
Block a user