forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into android-wear-integration
This commit is contained in:
commit
92a2530e0e
66 files changed
+3731
-1433
No files matched your search
@@ -0,0 +1,176 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Download management for the on-device transcription model (Moonshine, via
|
||||
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
|
||||
* demand into the same filesDir location that earlier releases copied the
|
||||
* bundled files to, so upgraded installs that already used on-device
|
||||
* transcription need no download. FaceclawVoiceController reads the files
|
||||
* from this directory.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
|
||||
* three files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODEL = {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
] as AsrModelFile[],
|
||||
totalBytes: 141300566,
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let downloader: any = null;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
let completedBytes = 0;
|
||||
let currentFileBytes = 0;
|
||||
const stateListeners = new Set<(state: AsrModelState) => void>();
|
||||
|
||||
function modelDirPath(): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
|
||||
}
|
||||
|
||||
function filePath(file: AsrModelFile): string {
|
||||
return `${modelDirPath()}/${file.name}`;
|
||||
}
|
||||
|
||||
function isFilePresent(file: AsrModelFile): boolean {
|
||||
try {
|
||||
const javaFile = new java.io.File(filePath(file));
|
||||
return javaFile.exists() && javaFile.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAsrModelReady(): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
return ASR_MODEL.files.every(isFilePresent);
|
||||
}
|
||||
|
||||
export function asrModelState(): AsrModelState {
|
||||
if (downloader) {
|
||||
return {
|
||||
status: "downloading",
|
||||
bytesDownloaded: completedBytes + currentFileBytes,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: isAsrModelReady() ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = asrModelState();
|
||||
stateListeners.forEach((listener) => listener(state));
|
||||
}
|
||||
|
||||
export function startAsrModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isAsrModelReady()) return;
|
||||
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile();
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
function downloadNextFile(): void {
|
||||
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
|
||||
if (!nextFile) {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
return;
|
||||
}
|
||||
currentFileBytes = 0;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
currentFileBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
},
|
||||
onDone: () => {
|
||||
completedBytes += nextFile.sizeBytes;
|
||||
currentFileBytes = 0;
|
||||
downloadNextFile();
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${ASR_MODEL.baseUrl}${nextFile.name}`,
|
||||
filePath(nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloader.start();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelAsrModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
export function deleteAsrModel(): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelAsrModelDownload();
|
||||
try {
|
||||
for (const file of ASR_MODEL.files) {
|
||||
new java.io.File(filePath(file)).delete();
|
||||
new java.io.File(`${filePath(file)}.part`).delete();
|
||||
}
|
||||
new java.io.File(modelDirPath()).delete();
|
||||
} catch (error) {
|
||||
console.error(`Voice model delete failed: ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
}
|
||||
+130
-28
@@ -1,12 +1,13 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { classifyAdvertisement, type RawAdvertisement } from "../g2/even-advertisement";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type DiscoveredDevice = {
|
||||
address: string;
|
||||
name: string;
|
||||
role: "left" | "right" | "ring";
|
||||
source: "paired" | "scan";
|
||||
export type DiscoveryEvents = {
|
||||
onAdvertisement: (raw: RawAdvertisement) => void;
|
||||
onScanFailed?: (code: number, message: string) => void;
|
||||
onLog?: (line: string) => void;
|
||||
};
|
||||
|
||||
export type DiscoveredAddressSet = {
|
||||
@@ -16,8 +17,16 @@ export type DiscoveredAddressSet = {
|
||||
summary: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* TS wrapper around the native FaceclawDeviceDiscovery. The live scan streams
|
||||
* raw advertisements (see `RawAdvertisement`); protocol decoding happens in
|
||||
* `app/g2/even-advertisement.ts` and pair grouping in
|
||||
* `app/g2/pairing-candidates.ts`.
|
||||
*/
|
||||
export class DeviceDiscoveryBridge {
|
||||
private readonly discovery: any;
|
||||
private listenerProxy: any = null;
|
||||
private events: DiscoveryEvents | null = null;
|
||||
|
||||
constructor() {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
@@ -25,40 +34,133 @@ export class DeviceDiscoveryBridge {
|
||||
this.discovery = new com.faceclaw.app.FaceclawDeviceDiscovery(context);
|
||||
}
|
||||
|
||||
async getBondedCandidates(): Promise<DiscoveredDevice[]> {
|
||||
return parseCandidates(String(this.discovery.getBondedCandidatesJson()));
|
||||
get bluetoothEnabled(): boolean {
|
||||
return !!this.discovery.isBluetoothEnabled();
|
||||
}
|
||||
|
||||
async scanCandidates(timeoutMs = 6000): Promise<DiscoveredDevice[]> {
|
||||
return parseCandidates(String(this.discovery.scanCandidatesJson(timeoutMs)));
|
||||
get scanning(): boolean {
|
||||
return !!this.discovery.isScanning();
|
||||
}
|
||||
|
||||
/** Start (or re-target) the live scan. Returns false when the radio is off. */
|
||||
startScan(events: DiscoveryEvents): boolean {
|
||||
this.events = events;
|
||||
if (!this.listenerProxy) {
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawDeviceDiscoveryListener({
|
||||
onAdvertisement: (json: string) => {
|
||||
const raw = parseRaw(String(json));
|
||||
if (raw) this.events?.onAdvertisement(raw);
|
||||
},
|
||||
onScanFailed: (code: number, message: string) => this.events?.onScanFailed?.(Number(code), String(message)),
|
||||
onLog: (line: string) => this.events?.onLog?.(String(line)),
|
||||
});
|
||||
this.discovery.setListener(this.listenerProxy);
|
||||
}
|
||||
return !!this.discovery.startScan();
|
||||
}
|
||||
|
||||
stopScan(): void {
|
||||
try {
|
||||
this.discovery.stopScan();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
/** Push bonded Even devices through the listener as `source: "paired"` entries. */
|
||||
emitBondedDevices(): void {
|
||||
if (!this.listenerProxy) return;
|
||||
this.discovery.emitBondedDevices();
|
||||
}
|
||||
|
||||
async getBondedCandidates(): Promise<RawAdvertisement[]> {
|
||||
return parseRawArray(String(this.discovery.getBondedCandidatesJson()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect advertisements for `timeoutMs` without blocking the UI thread
|
||||
* (scan callbacks are delivered on the main looper, so a blocking sleep
|
||||
* there would starve them). Latest sample per address wins.
|
||||
*/
|
||||
scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const latest = new Map<string, RawAdvertisement>();
|
||||
const started = this.startScan({
|
||||
onAdvertisement: (raw) => latest.set(raw.address, raw),
|
||||
onScanFailed: (code, message) => {
|
||||
this.stopScan();
|
||||
reject(new Error(`Bluetooth scan failed: ${message} (${code})`));
|
||||
},
|
||||
});
|
||||
if (!started) {
|
||||
reject(new Error("Bluetooth is off or BLE scanning is unavailable."));
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
this.stopScan();
|
||||
resolve(Array.from(latest.values()));
|
||||
}, Math.max(500, timeoutMs));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function buildAddressSet(candidates: DiscoveredDevice[]): DiscoveredAddressSet {
|
||||
const latestForRole = new Map<DiscoveredDevice["role"], DiscoveredDevice>();
|
||||
for (const candidate of candidates) {
|
||||
latestForRole.set(candidate.role, candidate);
|
||||
/**
|
||||
* Pick one address per role from a batch of raw advertisements — the manual
|
||||
* address page's "load" helper. The latest-seen device of each role wins;
|
||||
* the summary names the serial where one was advertised.
|
||||
*/
|
||||
export function buildAddressSet(candidates: RawAdvertisement[]): DiscoveredAddressSet {
|
||||
const classified = candidates.map(classifyAdvertisement).filter((item): item is NonNullable<typeof item> => !!item);
|
||||
const latestForRole = new Map<"left" | "right" | "ring", (typeof classified)[number]>();
|
||||
for (const candidate of classified) {
|
||||
const incumbent = latestForRole.get(candidate.role);
|
||||
if (!incumbent || incumbent.seenAtMs <= candidate.seenAtMs) latestForRole.set(candidate.role, candidate);
|
||||
}
|
||||
const selected = {
|
||||
const lines = classified.map((candidate) => {
|
||||
const extras = [candidate.serial ? `serial ${candidate.serial}` : null, candidate.rssi != null ? `${candidate.rssi} dBm` : null].filter(Boolean);
|
||||
return `${candidate.role}: ${candidate.name} ${candidate.address}${extras.length ? ` (${extras.join(", ")})` : ""}`;
|
||||
});
|
||||
return {
|
||||
left: latestForRole.get("left")?.address ?? "",
|
||||
right: latestForRole.get("right")?.address ?? "",
|
||||
ring: latestForRole.get("ring")?.address ?? "",
|
||||
};
|
||||
const lines = candidates.map((candidate) => `${candidate.role}: ${candidate.name} ${candidate.address}`);
|
||||
return {
|
||||
...selected,
|
||||
summary: lines.length ? lines.join("\n") : "No matching devices found.",
|
||||
};
|
||||
}
|
||||
|
||||
function parseCandidates(json: string): DiscoveredDevice[] {
|
||||
const raw = JSON.parse(json) as Array<Record<string, unknown>>;
|
||||
return raw
|
||||
.map((item) => ({
|
||||
address: String(item.address ?? ""),
|
||||
name: String(item.name ?? ""),
|
||||
role: String(item.role ?? "") as DiscoveredDevice["role"],
|
||||
source: String(item.source ?? "") as DiscoveredDevice["source"],
|
||||
}))
|
||||
.filter((item) => !!item.address && (item.role === "left" || item.role === "right" || item.role === "ring"));
|
||||
function parseRawArray(json: string): RawAdvertisement[] {
|
||||
const items = JSON.parse(json) as unknown[];
|
||||
return items.map((item) => parseRawObject(item)).filter((item): item is RawAdvertisement => !!item);
|
||||
}
|
||||
|
||||
function parseRaw(json: string): RawAdvertisement | null {
|
||||
try {
|
||||
return parseRawObject(JSON.parse(json));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseRawObject(item: unknown): RawAdvertisement | null {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
const record = item as Record<string, unknown>;
|
||||
const address = String(record.address ?? "");
|
||||
if (!address) return null;
|
||||
return {
|
||||
address,
|
||||
name: String(record.name ?? ""),
|
||||
manufacturerData: String(record.manufacturerData ?? ""),
|
||||
rssi: numberOrNull(record.rssi),
|
||||
txPower: numberOrNull(record.txPower),
|
||||
connectable: typeof record.connectable === "boolean" ? record.connectable : null,
|
||||
bonded: record.bonded === true,
|
||||
source: record.source === "paired" ? "paired" : "scan",
|
||||
seenAtMs: numberOrNull(record.seenAtMs) ?? Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
@@ -45,7 +45,6 @@ export type RawPcmListener = (pcm: Uint8Array) => void;
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private readonly speechEndListeners = new Set<() => void>();
|
||||
private controller: any | null = null;
|
||||
@@ -72,11 +71,6 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.statusListeners.delete(listener);
|
||||
}
|
||||
|
||||
onWakeWord(listener: (keyword: string) => void): () => void {
|
||||
this.wakeWordListeners.add(listener);
|
||||
return () => this.wakeWordListeners.delete(listener);
|
||||
}
|
||||
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcriptListeners.add(listener);
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
@@ -266,11 +260,6 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.rawActive) return;
|
||||
this.setStatus(String(status));
|
||||
},
|
||||
onWakeWord: (keyword: string) => {
|
||||
for (const listener of this.wakeWordListeners) {
|
||||
listener(String(keyword));
|
||||
}
|
||||
},
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user