Merge branch 'main' of https://github.com/jimrandomh/faceclaw into android-wear-integration

This commit is contained in:
James Babcock committed 2026-08-24 00:23:16 -07:00
commit 92a2530e0e
66 files changed
+3731 -1433

No files matched your search

+176
View File
@@ -0,0 +1,176 @@
import { Utils } from "@nativescript/core";
declare const com: any;
declare const java: any;
/**
* Download management for the on-device transcription model (Moonshine, via
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
* demand into the same filesDir location that earlier releases copied the
* bundled files to, so upgraded installs that already used on-device
* transcription need no download. FaceclawVoiceController reads the files
* from this directory.
*
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
* three files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file).
*/
type AsrModelFile = {
name: string;
sha256: string;
sizeBytes: number;
};
export const ASR_MODEL = {
label: "Moonshine (English)",
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
// this mirror serves the same files (sha256-verified) individually.
baseUrl:
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
files: [
{
name: "decoder_model_merged.ort",
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
sizeBytes: 109424400,
},
{
name: "encoder_model.ort",
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
sizeBytes: 31326816,
},
{
name: "tokens.txt",
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
sizeBytes: 549350,
},
] as AsrModelFile[],
totalBytes: 141300566,
};
export type AsrModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
let downloader: any = null;
// Bytes of files already fully downloaded in this run, plus progress within
// the file currently downloading; drives the aggregate percentage.
let completedBytes = 0;
let currentFileBytes = 0;
const stateListeners = new Set<(state: AsrModelState) => void>();
function modelDirPath(): string {
const context = Utils.android.getApplicationContext();
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
}
function filePath(file: AsrModelFile): string {
return `${modelDirPath()}/${file.name}`;
}
function isFilePresent(file: AsrModelFile): boolean {
try {
const javaFile = new java.io.File(filePath(file));
return javaFile.exists() && javaFile.length() > 0;
} catch {
return false;
}
}
export function isAsrModelReady(): boolean {
if (!global.isAndroid) return false;
return ASR_MODEL.files.every(isFilePresent);
}
export function asrModelState(): AsrModelState {
if (downloader) {
return {
status: "downloading",
bytesDownloaded: completedBytes + currentFileBytes,
totalBytes: ASR_MODEL.totalBytes,
};
}
return {
status: isAsrModelReady() ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: ASR_MODEL.totalBytes,
};
}
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
}
function notifyStateChanged(): void {
const state = asrModelState();
stateListeners.forEach((listener) => listener(state));
}
export function startAsrModelDownload(): void {
if (!global.isAndroid || downloader || isAsrModelReady()) return;
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
downloadNextFile();
notifyStateChanged();
}
function downloadNextFile(): void {
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
if (!nextFile) {
downloader = null;
notifyStateChanged();
return;
}
currentFileBytes = 0;
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
currentFileBytes = Number(bytes);
notifyStateChanged();
},
onDone: () => {
completedBytes += nextFile.sizeBytes;
currentFileBytes = 0;
downloadNextFile();
},
onError: (message: string) => {
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
downloader = null;
notifyStateChanged();
},
});
downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${ASR_MODEL.baseUrl}${nextFile.name}`,
filePath(nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloader.start();
}
/** Stops the download; already-fetched bytes are kept and resumed next time. */
export function cancelAsrModelDownload(): void {
if (!downloader) return;
downloader.cancel();
downloader = null;
notifyStateChanged();
}
export function deleteAsrModel(): void {
if (!global.isAndroid) return;
cancelAsrModelDownload();
try {
for (const file of ASR_MODEL.files) {
new java.io.File(filePath(file)).delete();
new java.io.File(`${filePath(file)}.part`).delete();
}
new java.io.File(modelDirPath()).delete();
} catch (error) {
console.error(`Voice model delete failed: ${String(error)}`);
}
notifyStateChanged();
}
+130 -28
View File
@@ -1,12 +1,13 @@
import { Utils } from "@nativescript/core";
import { classifyAdvertisement, type RawAdvertisement } from "../g2/even-advertisement";
declare const com: any;
export type DiscoveredDevice = {
address: string;
name: string;
role: "left" | "right" | "ring";
source: "paired" | "scan";
export type DiscoveryEvents = {
onAdvertisement: (raw: RawAdvertisement) => void;
onScanFailed?: (code: number, message: string) => void;
onLog?: (line: string) => void;
};
export type DiscoveredAddressSet = {
@@ -16,8 +17,16 @@ export type DiscoveredAddressSet = {
summary: string;
};
/**
* TS wrapper around the native FaceclawDeviceDiscovery. The live scan streams
* raw advertisements (see `RawAdvertisement`); protocol decoding happens in
* `app/g2/even-advertisement.ts` and pair grouping in
* `app/g2/pairing-candidates.ts`.
*/
export class DeviceDiscoveryBridge {
private readonly discovery: any;
private listenerProxy: any = null;
private events: DiscoveryEvents | null = null;
constructor() {
const context = Utils.android.getApplicationContext();
@@ -25,40 +34,133 @@ export class DeviceDiscoveryBridge {
this.discovery = new com.faceclaw.app.FaceclawDeviceDiscovery(context);
}
async getBondedCandidates(): Promise<DiscoveredDevice[]> {
return parseCandidates(String(this.discovery.getBondedCandidatesJson()));
get bluetoothEnabled(): boolean {
return !!this.discovery.isBluetoothEnabled();
}
async scanCandidates(timeoutMs = 6000): Promise<DiscoveredDevice[]> {
return parseCandidates(String(this.discovery.scanCandidatesJson(timeoutMs)));
get scanning(): boolean {
return !!this.discovery.isScanning();
}
/** Start (or re-target) the live scan. Returns false when the radio is off. */
startScan(events: DiscoveryEvents): boolean {
this.events = events;
if (!this.listenerProxy) {
this.listenerProxy = new com.faceclaw.app.FaceclawDeviceDiscoveryListener({
onAdvertisement: (json: string) => {
const raw = parseRaw(String(json));
if (raw) this.events?.onAdvertisement(raw);
},
onScanFailed: (code: number, message: string) => this.events?.onScanFailed?.(Number(code), String(message)),
onLog: (line: string) => this.events?.onLog?.(String(line)),
});
this.discovery.setListener(this.listenerProxy);
}
return !!this.discovery.startScan();
}
stopScan(): void {
try {
this.discovery.stopScan();
} catch {
// ignore
}
}
/** Push bonded Even devices through the listener as `source: "paired"` entries. */
emitBondedDevices(): void {
if (!this.listenerProxy) return;
this.discovery.emitBondedDevices();
}
async getBondedCandidates(): Promise<RawAdvertisement[]> {
return parseRawArray(String(this.discovery.getBondedCandidatesJson()));
}
/**
* Collect advertisements for `timeoutMs` without blocking the UI thread
* (scan callbacks are delivered on the main looper, so a blocking sleep
* there would starve them). Latest sample per address wins.
*/
scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
return new Promise((resolve, reject) => {
const latest = new Map<string, RawAdvertisement>();
const started = this.startScan({
onAdvertisement: (raw) => latest.set(raw.address, raw),
onScanFailed: (code, message) => {
this.stopScan();
reject(new Error(`Bluetooth scan failed: ${message} (${code})`));
},
});
if (!started) {
reject(new Error("Bluetooth is off or BLE scanning is unavailable."));
return;
}
setTimeout(() => {
this.stopScan();
resolve(Array.from(latest.values()));
}, Math.max(500, timeoutMs));
});
}
}
export function buildAddressSet(candidates: DiscoveredDevice[]): DiscoveredAddressSet {
const latestForRole = new Map<DiscoveredDevice["role"], DiscoveredDevice>();
for (const candidate of candidates) {
latestForRole.set(candidate.role, candidate);
/**
* Pick one address per role from a batch of raw advertisements — the manual
* address page's "load" helper. The latest-seen device of each role wins;
* the summary names the serial where one was advertised.
*/
export function buildAddressSet(candidates: RawAdvertisement[]): DiscoveredAddressSet {
const classified = candidates.map(classifyAdvertisement).filter((item): item is NonNullable<typeof item> => !!item);
const latestForRole = new Map<"left" | "right" | "ring", (typeof classified)[number]>();
for (const candidate of classified) {
const incumbent = latestForRole.get(candidate.role);
if (!incumbent || incumbent.seenAtMs <= candidate.seenAtMs) latestForRole.set(candidate.role, candidate);
}
const selected = {
const lines = classified.map((candidate) => {
const extras = [candidate.serial ? `serial ${candidate.serial}` : null, candidate.rssi != null ? `${candidate.rssi} dBm` : null].filter(Boolean);
return `${candidate.role}: ${candidate.name} ${candidate.address}${extras.length ? ` (${extras.join(", ")})` : ""}`;
});
return {
left: latestForRole.get("left")?.address ?? "",
right: latestForRole.get("right")?.address ?? "",
ring: latestForRole.get("ring")?.address ?? "",
};
const lines = candidates.map((candidate) => `${candidate.role}: ${candidate.name} ${candidate.address}`);
return {
...selected,
summary: lines.length ? lines.join("\n") : "No matching devices found.",
};
}
function parseCandidates(json: string): DiscoveredDevice[] {
const raw = JSON.parse(json) as Array<Record<string, unknown>>;
return raw
.map((item) => ({
address: String(item.address ?? ""),
name: String(item.name ?? ""),
role: String(item.role ?? "") as DiscoveredDevice["role"],
source: String(item.source ?? "") as DiscoveredDevice["source"],
}))
.filter((item) => !!item.address && (item.role === "left" || item.role === "right" || item.role === "ring"));
function parseRawArray(json: string): RawAdvertisement[] {
const items = JSON.parse(json) as unknown[];
return items.map((item) => parseRawObject(item)).filter((item): item is RawAdvertisement => !!item);
}
function parseRaw(json: string): RawAdvertisement | null {
try {
return parseRawObject(JSON.parse(json));
} catch {
return null;
}
}
function parseRawObject(item: unknown): RawAdvertisement | null {
if (!item || typeof item !== "object") return null;
const record = item as Record<string, unknown>;
const address = String(record.address ?? "");
if (!address) return null;
return {
address,
name: String(record.name ?? ""),
manufacturerData: String(record.manufacturerData ?? ""),
rssi: numberOrNull(record.rssi),
txPower: numberOrNull(record.txPower),
connectable: typeof record.connectable === "boolean" ? record.connectable : null,
bonded: record.bonded === true,
source: record.source === "paired" ? "paired" : "scan",
seenAtMs: numberOrNull(record.seenAtMs) ?? Date.now(),
};
}
function numberOrNull(value: unknown): number | null {
if (value === null || value === undefined) return null;
const n = Number(value);
return Number.isFinite(n) ? n : null;
}
-11
View File
@@ -45,7 +45,6 @@ export type RawPcmListener = (pcm: Uint8Array) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private readonly speechEndListeners = new Set<() => void>();
private controller: any | null = null;
@@ -72,11 +71,6 @@ export class FaceclawVoiceControlBridge {
return () => this.statusListeners.delete(listener);
}
onWakeWord(listener: (keyword: string) => void): () => void {
this.wakeWordListeners.add(listener);
return () => this.wakeWordListeners.delete(listener);
}
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
this.transcriptListeners.add(listener);
return () => this.transcriptListeners.delete(listener);
@@ -266,11 +260,6 @@ export class FaceclawVoiceControlBridge {
if (this.rawActive) return;
this.setStatus(String(status));
},
onWakeWord: (keyword: string) => {
for (const listener of this.wakeWordListeners) {
listener(String(keyword));
}
},
onTranscript: (text: string, isFinal: boolean) => {
this.emitTranscript(String(text), Boolean(isFinal));
},