forked from Blendman974/faceclaw
Add mic support for EvenHub apps
This commit is contained in:
1 parent
add8763028
commit
b86f395866
11 files changed
+593
-17
No files matched your search
@@ -10,6 +10,7 @@
|
||||
* an integrity check only, not a signature; we skip verifying it.
|
||||
*/
|
||||
import { decompress } from "fzstd";
|
||||
import { parsePermissions, type EvenHubPermission } from "./permissions";
|
||||
|
||||
const XOR_KEY = "EVEN REALITIES";
|
||||
|
||||
@@ -31,6 +32,8 @@ export type EvenHubManifest = {
|
||||
entrypoint: string;
|
||||
/** Hosts the app declares it will reach; not yet enforced. */
|
||||
networkWhitelist: string[];
|
||||
/** Normalized declared permissions (both manifest shapes). */
|
||||
permissions: EvenHubPermission[];
|
||||
raw: Record<string, unknown>;
|
||||
};
|
||||
|
||||
@@ -115,19 +118,13 @@ export function parseManifest(appJsonText: string): EvenHubManifest {
|
||||
const version = String(raw.version ?? "0");
|
||||
const entrypoint = String(raw.entrypoint ?? "index.html");
|
||||
|
||||
const permissions = parsePermissions(raw.permissions);
|
||||
const networkWhitelist: string[] = [];
|
||||
const permissions = raw.permissions;
|
||||
if (Array.isArray(permissions)) {
|
||||
for (const perm of permissions) {
|
||||
if (perm && typeof perm === "object" && (perm as Record<string, unknown>).name === "network") {
|
||||
const whitelist = (perm as Record<string, unknown>).whitelist;
|
||||
if (Array.isArray(whitelist)) networkWhitelist.push(...whitelist.map(String));
|
||||
}
|
||||
for (const permission of permissions) {
|
||||
if (permission.name === "network" && permission.whitelist) {
|
||||
networkWhitelist.push(...permission.whitelist);
|
||||
}
|
||||
} else if (permissions && typeof permissions === "object") {
|
||||
const network = (permissions as Record<string, unknown>).network;
|
||||
if (Array.isArray(network)) networkWhitelist.push(...network.map(String));
|
||||
}
|
||||
|
||||
return { packageId, name, version, entrypoint, networkWhitelist, raw };
|
||||
return { packageId, name, version, entrypoint, networkWhitelist, permissions, raw };
|
||||
}
|
||||
@@ -77,5 +77,12 @@ export function createEvenHubWindow(
|
||||
session.setForeground(foreground);
|
||||
};
|
||||
|
||||
// Screen on/off gates the app's microphone (silent while the screen is off).
|
||||
const baseSetScreenOn = created.window.setScreenOn;
|
||||
created.window.setScreenOn = (on) => {
|
||||
baseSetScreenOn?.(on);
|
||||
session.setScreenOn(on);
|
||||
};
|
||||
|
||||
return created;
|
||||
}
|
||||
@@ -142,6 +142,21 @@ export function installEvenHubPackageFile(path: string): InstalledEvenHubApp {
|
||||
return installEvenHubPackageBytes(bytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an .ehpk's manifest without installing it — used by the Files app to
|
||||
* show the permission-confirmation dialog before install/run. Returns null if
|
||||
* the file can't be read or parsed.
|
||||
*/
|
||||
export function readEvenHubPackageManifest(path: string): EvenHubManifest | null {
|
||||
const bytes = readBinaryFile(path);
|
||||
if (!bytes) return null;
|
||||
try {
|
||||
return inspectEhpk(bytes).manifest;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove the stored EHPK, unpacked runtime files, and registry entry. */
|
||||
export function uninstallEvenHubPackage(packageId: string): boolean {
|
||||
const apps = getInstalledEvenHubApps();
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Arbitrates the single G2 microphone among running EvenHub apps.
|
||||
*
|
||||
* An app asks for the mic with audioControl(true); it only actually receives
|
||||
* audio while it is the foreground (visible) window, the screen is on, and no
|
||||
* voice-input modal (the assistant) is active. When any of those stops holding,
|
||||
* the app simply stops receiving frames — it is not told, matching the stock
|
||||
* contract where a backgrounded app's mic goes silent.
|
||||
*
|
||||
* Because only one window is foreground at a time, at most one requesting app
|
||||
* is ever eligible. The router runs the shared decode-only raw-PCM tap
|
||||
* (voiceControlBridge) while an eligible app exists and routes each decoded
|
||||
* frame to it; STT sessions preempt the tap at the bridge level.
|
||||
*/
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type EvenHubMicClient = {
|
||||
readonly windowId: string;
|
||||
isForeground(): boolean;
|
||||
isScreenOn(): boolean;
|
||||
deliverAudioPcm(pcm: Uint8Array): void;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
class EvenHubMicRouter {
|
||||
private readonly requesting = new Set<EvenHubMicClient>();
|
||||
private active: EvenHubMicClient | null = null;
|
||||
private unsubscribeRawPcm: (() => void) | null = null;
|
||||
private unsubscribeVoiceActivity: (() => void) | null = null;
|
||||
|
||||
/** An app enabled its microphone (audioControl true). */
|
||||
requestMic(client: EvenHubMicClient): void {
|
||||
this.requesting.add(client);
|
||||
this.ensureSubscriptions();
|
||||
this.evaluate();
|
||||
}
|
||||
|
||||
/** An app disabled its microphone, backgrounded, or closed. */
|
||||
releaseMic(client: EvenHubMicClient): void {
|
||||
if (!this.requesting.delete(client)) return;
|
||||
this.evaluate();
|
||||
}
|
||||
|
||||
/** A requesting app's foreground/screen state changed. */
|
||||
notifyEligibilityChanged(): void {
|
||||
this.evaluate();
|
||||
}
|
||||
|
||||
private ensureSubscriptions(): void {
|
||||
if (!this.unsubscribeRawPcm) {
|
||||
this.unsubscribeRawPcm = voiceControlBridge.onRawPcm((pcm) => {
|
||||
this.active?.deliverAudioPcm(pcm);
|
||||
});
|
||||
}
|
||||
if (!this.unsubscribeVoiceActivity) {
|
||||
this.unsubscribeVoiceActivity = voiceActivity.subscribe(() => this.evaluate());
|
||||
}
|
||||
}
|
||||
|
||||
private eligible(): EvenHubMicClient | null {
|
||||
if (voiceActivity.isActive()) return null;
|
||||
return Array.from(this.requesting).find(
|
||||
(client) => client.isForeground() && client.isScreenOn(),
|
||||
) ?? null;
|
||||
}
|
||||
|
||||
private evaluate(): void {
|
||||
const next = this.eligible();
|
||||
this.active = next;
|
||||
if (next) {
|
||||
// Idempotent: no-op if already running, and re-asserted on every
|
||||
// eligibility change so it self-heals after an STT preemption. Returns
|
||||
// false if STT currently owns the mic; a later trigger (assistant modal
|
||||
// close, foreground/screen change) re-tries.
|
||||
voiceControlBridge.startRawCapture({ communicator: activeCommunicator() });
|
||||
} else {
|
||||
voiceControlBridge.stopRawCapture();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const evenHubMicRouter = new EvenHubMicRouter();
|
||||
@@ -0,0 +1,117 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { truncateText } from "../../graphics/textwrap";
|
||||
import { Layer, type DashboardInputEvent, type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { drawSelectionHighlight } from "../../ui/menu";
|
||||
import { permissionDetail, permissionLabel, type EvenHubPermission } from "./permissions";
|
||||
|
||||
const DIALOG_X = 8;
|
||||
const DIALOG_Y = 8;
|
||||
const DIALOG_WIDTH = 272;
|
||||
const PADDING = 10;
|
||||
const HEADER_STEP = 16;
|
||||
const PERM_LABEL_STEP = 15;
|
||||
const PERM_DETAIL_STEP = 13;
|
||||
const PERM_GAP = 4;
|
||||
const ACTION_ROW_HEIGHT = 20;
|
||||
|
||||
/**
|
||||
* Confirmation dialog listing the permissions an EvenHub app declares, shown
|
||||
* before installing it or running an uninstalled package. Allow proceeds;
|
||||
* Cancel (or double-click) backs out. Apps that declare no permissions never
|
||||
* reach this dialog — the caller runs them directly.
|
||||
*/
|
||||
export class EvenHubPermissionDialogLayer implements Layer {
|
||||
/** 0 = Allow, 1 = Cancel. */
|
||||
private selectedIndex = 0;
|
||||
private resolved = false;
|
||||
|
||||
constructor(
|
||||
private readonly appName: string,
|
||||
private readonly permissions: EvenHubPermission[],
|
||||
private readonly onConfirm: () => void,
|
||||
private readonly onCancel: () => void,
|
||||
) {}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const { height: viewportHeight } = ctx.stack.getBaseSize();
|
||||
const image = paintBelow();
|
||||
const textWidth = DIALOG_WIDTH - 2 * PADDING - 4;
|
||||
const left = DIALOG_X + PADDING + 2;
|
||||
|
||||
const permsHeight = this.permissions.reduce(
|
||||
(sum, permission) => sum + PERM_LABEL_STEP + (permissionDetail(permission) ? PERM_DETAIL_STEP : 0) + PERM_GAP,
|
||||
0,
|
||||
);
|
||||
const bodyTop = PADDING + HEADER_STEP + 6 + permsHeight + 6;
|
||||
const height = Math.min(bodyTop + 2 * ACTION_ROW_HEIGHT + PADDING, viewportHeight - 2 * DIALOG_Y);
|
||||
|
||||
// Fill 1 (transparent color key is 0), outline for the dialog edge.
|
||||
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 1);
|
||||
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 72);
|
||||
|
||||
let y = DIALOG_Y + PADDING;
|
||||
image.drawText(font, left, y, truncateText(font, `${this.appName} needs:`, textWidth), 235);
|
||||
y += HEADER_STEP + 6;
|
||||
|
||||
for (const permission of this.permissions) {
|
||||
image.drawText(font, left, y, truncateText(font, permissionLabel(permission.name), textWidth), 220);
|
||||
y += PERM_LABEL_STEP;
|
||||
const detail = permissionDetail(permission);
|
||||
if (detail) {
|
||||
image.drawText(font, left + 8, y, truncateText(font, detail, textWidth - 8), 140);
|
||||
y += PERM_DETAIL_STEP;
|
||||
}
|
||||
y += PERM_GAP;
|
||||
}
|
||||
y += 2;
|
||||
|
||||
const focused = ctx.stack.isFocused();
|
||||
const actions = ["Allow", "Cancel"];
|
||||
for (let index = 0; index < actions.length; index++) {
|
||||
const rowY = y + index * ACTION_ROW_HEIGHT;
|
||||
const selected = index === this.selectedIndex;
|
||||
if (selected) {
|
||||
drawSelectionHighlight(image, DIALOG_X + 12, rowY, DIALOG_WIDTH - 24, ACTION_ROW_HEIGHT - 1, focused, 8);
|
||||
}
|
||||
image.drawText(font, DIALOG_X + 22, rowY + 3, actions[index]!, selected ? 255 : 200);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.selectedIndex = (this.selectedIndex + 1) % 2;
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.selectedIndex = (this.selectedIndex + 1) % 2;
|
||||
return;
|
||||
case "click":
|
||||
this.resolve(ctx, this.selectedIndex === 0);
|
||||
return;
|
||||
case "double-click":
|
||||
this.resolve(ctx, false);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
// Dismissed some other way (window closed, back button): treat as cancel.
|
||||
if (!this.resolved) {
|
||||
this.resolved = true;
|
||||
this.onCancel();
|
||||
}
|
||||
}
|
||||
|
||||
private resolve(ctx: LayerContext, confirmed: boolean): void {
|
||||
if (this.resolved) return;
|
||||
this.resolved = true;
|
||||
ctx.stack.pop();
|
||||
if (confirmed) this.onConfirm();
|
||||
else this.onCancel();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* EvenHub app permissions: parsing the manifest's declared permissions and
|
||||
* turning them into the human-readable list Faceclaw shows for confirmation.
|
||||
*
|
||||
* Two manifest shapes exist in the wild (see notes/evenhub_compatibility.txt):
|
||||
* - Current SDK: `permissions` is an array of {name, desc?, whitelist?}, with
|
||||
* names network, location, g2-microphone, phone-microphone, album, camera.
|
||||
* - Older shape: `permissions` is a map {"network": [...hosts], "fs": [...]}.
|
||||
*
|
||||
* Permissions are confirmed by the user when an app is installed, or when an
|
||||
* uninstalled .ehpk is run directly (installing implies a prior confirmation).
|
||||
* Once past that gate the running app is treated as having been granted every
|
||||
* permission it declared; runtime surfaces (e.g. the microphone) check the
|
||||
* declared set to decide whether to honor a request.
|
||||
*/
|
||||
|
||||
export type EvenHubPermissionName =
|
||||
| "network"
|
||||
| "location"
|
||||
| "g2-microphone"
|
||||
| "phone-microphone"
|
||||
| "album"
|
||||
| "camera"
|
||||
| "fs"
|
||||
| string;
|
||||
|
||||
export type EvenHubPermission = {
|
||||
name: EvenHubPermissionName;
|
||||
/** App-supplied description, when the manifest provides one. */
|
||||
description?: string;
|
||||
/** For network/fs: the declared host or path whitelist. */
|
||||
whitelist?: string[];
|
||||
};
|
||||
|
||||
/** The mic permission names an audioControl request may be gated behind. */
|
||||
const MICROPHONE_PERMISSIONS = new Set(["g2-microphone", "phone-microphone", "microphone"]);
|
||||
|
||||
/** Short label shown as the permission's title in the confirmation dialog. */
|
||||
export function permissionLabel(name: EvenHubPermissionName): string {
|
||||
switch (name) {
|
||||
case "network":
|
||||
return "Network access";
|
||||
case "location":
|
||||
return "Location";
|
||||
case "g2-microphone":
|
||||
return "Glasses microphone";
|
||||
case "phone-microphone":
|
||||
return "Phone microphone";
|
||||
case "microphone":
|
||||
return "Microphone";
|
||||
case "album":
|
||||
return "Photo album";
|
||||
case "camera":
|
||||
return "Camera";
|
||||
case "fs":
|
||||
return "Bundled files";
|
||||
default:
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
/** Secondary line under the label: the app's own description, or a default. */
|
||||
export function permissionDetail(permission: EvenHubPermission): string {
|
||||
if (permission.description && permission.description.trim()) {
|
||||
return permission.description.trim();
|
||||
}
|
||||
if ((permission.name === "network" || permission.name === "fs") && permission.whitelist?.length) {
|
||||
return permission.whitelist.join(", ");
|
||||
}
|
||||
switch (permission.name) {
|
||||
case "network":
|
||||
return "Reach servers on the internet";
|
||||
case "location":
|
||||
return "Read your location";
|
||||
case "g2-microphone":
|
||||
return "Listen through the glasses microphone";
|
||||
case "phone-microphone":
|
||||
return "Listen through the phone microphone";
|
||||
case "microphone":
|
||||
return "Listen through the microphone";
|
||||
case "album":
|
||||
return "Read photos from your album";
|
||||
case "camera":
|
||||
return "Capture photos from the camera";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function isMicrophonePermission(name: EvenHubPermissionName): boolean {
|
||||
return MICROPHONE_PERMISSIONS.has(name);
|
||||
}
|
||||
|
||||
/** Whether the declared set includes any microphone permission. */
|
||||
export function permissionsIncludeMicrophone(permissions: EvenHubPermission[]): boolean {
|
||||
return permissions.some((permission) => isMicrophonePermission(permission.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the manifest's raw `permissions` value into a normalized list,
|
||||
* accepting both the array and the older map shape. Unknown names pass through
|
||||
* so the confirmation dialog still surfaces them.
|
||||
*/
|
||||
export function parsePermissions(raw: unknown): EvenHubPermission[] {
|
||||
const out: EvenHubPermission[] = [];
|
||||
const seen = new Set<string>();
|
||||
const add = (permission: EvenHubPermission) => {
|
||||
const name = String(permission.name).trim();
|
||||
if (!name || seen.has(name)) return;
|
||||
seen.add(name);
|
||||
out.push({ ...permission, name });
|
||||
};
|
||||
|
||||
if (Array.isArray(raw)) {
|
||||
for (const entry of raw) {
|
||||
if (typeof entry === "string") {
|
||||
add({ name: entry });
|
||||
} else if (entry && typeof entry === "object") {
|
||||
const record = entry as Record<string, unknown>;
|
||||
const name = record.name;
|
||||
if (typeof name !== "string" || !name.trim()) continue;
|
||||
const description = typeof record.desc === "string"
|
||||
? record.desc
|
||||
: typeof record.description === "string"
|
||||
? record.description
|
||||
: undefined;
|
||||
const whitelist = Array.isArray(record.whitelist)
|
||||
? record.whitelist.map(String)
|
||||
: undefined;
|
||||
add({ name, description, whitelist });
|
||||
}
|
||||
}
|
||||
} else if (raw && typeof raw === "object") {
|
||||
// Older map shape: {"network": ["host", ...], "fs": ["./assets"]}.
|
||||
for (const [name, value] of Object.entries(raw as Record<string, unknown>)) {
|
||||
const whitelist = Array.isArray(value) ? value.map(String) : undefined;
|
||||
add({ name, whitelist });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A stable signature of a granted permission set, for comparing what the user
|
||||
* confirmed against what a later version of the app declares.
|
||||
*/
|
||||
export function permissionsSignature(permissions: EvenHubPermission[]): string {
|
||||
return permissions
|
||||
.map((permission) => {
|
||||
const list = permission.whitelist?.length ? `(${[...permission.whitelist].sort().join(",")})` : "";
|
||||
return `${permission.name}${list}`;
|
||||
})
|
||||
.sort()
|
||||
.join(";");
|
||||
}
|
||||
@@ -34,6 +34,8 @@ import {
|
||||
} from "./containers";
|
||||
import { compositePage } from "./compositor";
|
||||
import { type EvenHubManifest } from "./ehpk";
|
||||
import { permissionsIncludeMicrophone } from "./permissions";
|
||||
import { evenHubMicRouter, type EvenHubMicClient } from "./mic-router";
|
||||
|
||||
const UPNG = require("upng-js");
|
||||
|
||||
@@ -166,7 +168,7 @@ export type EvenHubWindowHooks = {
|
||||
focusSwitcher: () => void;
|
||||
};
|
||||
|
||||
export class EvenHubSession {
|
||||
export class EvenHubSession implements EvenHubMicClient {
|
||||
readonly manifest: EvenHubManifest;
|
||||
readonly distDir: string;
|
||||
|
||||
@@ -179,6 +181,10 @@ export class EvenHubSession {
|
||||
private systemExitSent = false;
|
||||
/** Shell focus state; true from launch (apps open focused). */
|
||||
private foreground = true;
|
||||
/** Phone screen state; the mic goes silent while the screen is off. */
|
||||
private screenOn = true;
|
||||
/** The app has enabled its microphone via audioControl(true). */
|
||||
private micRequested = false;
|
||||
private log: (message: string) => void;
|
||||
|
||||
constructor(manifest: EvenHubManifest, distDir: string, log: (message: string) => void) {
|
||||
@@ -288,10 +294,66 @@ export class EvenHubSession {
|
||||
setForeground(foreground: boolean): void {
|
||||
if (this.foreground === foreground) return;
|
||||
this.foreground = foreground;
|
||||
// Foreground gates mic eligibility (an app only hears audio while visible).
|
||||
if (this.micRequested) evenHubMicRouter.notifyEligibilityChanged();
|
||||
if (!this.pageCreated) return;
|
||||
this.emitSysEvent(foreground ? FOREGROUND_ENTER_EVENT : FOREGROUND_EXIT_EVENT, 0);
|
||||
}
|
||||
|
||||
/** Screen turned on/off (forwarded from the shell); mutes the mic when off. */
|
||||
setScreenOn(on: boolean): void {
|
||||
if (this.screenOn === on) return;
|
||||
this.screenOn = on;
|
||||
if (this.micRequested) evenHubMicRouter.notifyEligibilityChanged();
|
||||
}
|
||||
|
||||
// ----- microphone (EvenHubMicClient) -----
|
||||
|
||||
get windowId(): string {
|
||||
return this.manifest.packageId;
|
||||
}
|
||||
|
||||
isForeground(): boolean {
|
||||
return this.foreground;
|
||||
}
|
||||
|
||||
isScreenOn(): boolean {
|
||||
return this.screenOn;
|
||||
}
|
||||
|
||||
/** A decoded mic frame from the router: hand it to the app as an audioEvent. */
|
||||
deliverAudioPcm(pcm: Uint8Array): void {
|
||||
const audioPcm = Array.from(pcm);
|
||||
this.pushMessage("evenHubEvent", { type: "audioEvent", jsonData: { audioPcm } });
|
||||
}
|
||||
|
||||
/**
|
||||
* audioControl(enable, source?): open or close the microphone for this app.
|
||||
* Gated on a declared microphone permission (confirmed at install/first run).
|
||||
* Enabling only registers intent — the router decides when audio actually
|
||||
* flows (foreground + screen on + no assistant modal).
|
||||
*/
|
||||
private audioControl(data: Record<string, unknown>): boolean {
|
||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||
return false;
|
||||
}
|
||||
const enable = readAudioEnable(data);
|
||||
// The 0.0.12 payload shape isn't captured; log it once per toggle so the
|
||||
// real field names can be confirmed from hardware.
|
||||
this.log(`evenhub: audioControl enable=${enable} data=${JSON.stringify(data)}`);
|
||||
if (enable) {
|
||||
if (!this.micRequested) {
|
||||
this.micRequested = true;
|
||||
evenHubMicRouter.requestMic(this);
|
||||
}
|
||||
} else if (this.micRequested) {
|
||||
this.micRequested = false;
|
||||
evenHubMicRouter.releaseMic(this);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ----- bridge calls (web -> host) -----
|
||||
|
||||
/**
|
||||
@@ -351,8 +413,9 @@ export class EvenHubSession {
|
||||
isInCase: false,
|
||||
},
|
||||
};
|
||||
// Unsupported surface (mic, IMU, location, pickers): fail politely.
|
||||
case "audioControl":
|
||||
return this.audioControl(data);
|
||||
// Unsupported surface (IMU, location, pickers): fail politely.
|
||||
case "imuControl":
|
||||
case "startAppLocationUpdates":
|
||||
case "stopAppLocationUpdates":
|
||||
@@ -497,6 +560,10 @@ export class EvenHubSession {
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
if (this.micRequested) {
|
||||
this.micRequested = false;
|
||||
evenHubMicRouter.releaseMic(this);
|
||||
}
|
||||
if (!this.systemExitSent) {
|
||||
// Best effort: lets the app's teardown handlers run before the webview dies.
|
||||
this.emitSysEvent(SYSTEM_EXIT_EVENT, 0);
|
||||
@@ -522,6 +589,22 @@ function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the enable flag from an audioControl payload. The exact wire field name
|
||||
* for SDK 0.0.12 isn't captured, so accept the plausible spellings; default to
|
||||
* enabling (the common intent) when the payload is opaque, and log it so the
|
||||
* real shape can be pinned from hardware.
|
||||
*/
|
||||
function readAudioEnable(data: Record<string, unknown>): boolean {
|
||||
for (const key of ["enable", "open", "isOpen", "on", "status", "value", "start"]) {
|
||||
const value = data[key];
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return value !== 0;
|
||||
if (typeof value === "string") return value === "true" || value === "1" || value === "on" || value === "open";
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** ring/left-arm/right-arm -> PB EventSourceType. */
|
||||
function gestureSource(source: "ring" | "left-arm" | "right-arm"): number {
|
||||
switch (source) {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
import { FileBrowserLayer } from "./file-browser";
|
||||
import { FileInfoDialogLayer, type FileInfoAction } from "./file-info-dialog";
|
||||
import { type LayerContext } from "../../ui/layers";
|
||||
import { FontPreviewerLayer } from "./font-previewer";
|
||||
import { ImageViewerLayer } from "./image-viewer";
|
||||
import { TextViewerLayer } from "./text-viewer";
|
||||
@@ -138,6 +141,22 @@ export function createImageDocumentWindow(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Show the permission-confirmation dialog for an .ehpk before install/run,
|
||||
* then run `proceed` if the user allows. Packages that declare no permissions
|
||||
* skip the dialog and proceed immediately.
|
||||
*/
|
||||
function confirmEhpkPermissions(ctx: LayerContext, entry: DirectoryEntry, proceed: () => void): void {
|
||||
const manifest = readEvenHubPackageManifest(entry.path);
|
||||
if (!manifest || manifest.permissions.length === 0) {
|
||||
proceed();
|
||||
return;
|
||||
}
|
||||
ctx.stack.push(
|
||||
new EvenHubPermissionDialogLayer(manifest.name, manifest.permissions, proceed, () => {}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The open actions for the picked-file dialog: View here / Open in new
|
||||
* window for viewable types, empty for everything else (metadata only).
|
||||
@@ -169,14 +188,16 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
label: "Run app",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openEhpkApp(entry.path);
|
||||
confirmEhpkPermissions(ctx, entry, () => options.openEhpkApp(entry.path));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Install",
|
||||
onSelect: async (ctx) => {
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
await options.installEhpkApp(entry.path);
|
||||
confirmEhpkPermissions(ctx, entry, () => {
|
||||
void options.installEhpkApp(entry.path);
|
||||
});
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -41,6 +41,8 @@ export type PushToTalkOptions = {
|
||||
/** Who currently wants the mic running. */
|
||||
type CaptureHolder = "ptt" | "continuous";
|
||||
|
||||
export type RawPcmListener = (pcm: Uint8Array) => void;
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
@@ -57,6 +59,12 @@ export class FaceclawVoiceControlBridge {
|
||||
private readonly captureHolders = new Set<CaptureHolder>();
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the
|
||||
// decode-only "cloud" mode and every decoded PCM frame is broadcast to these
|
||||
// listeners. STT capture preempts it — a live assistant/transcribe session
|
||||
// owns the single G2 mic and the raw tap is stopped for its duration.
|
||||
private readonly rawPcmListeners = new Set<RawPcmListener>();
|
||||
private rawActive = false;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -102,8 +110,46 @@ export class FaceclawVoiceControlBridge {
|
||||
this.releaseCapture("continuous", false);
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
onRawPcm(listener: RawPcmListener): () => void {
|
||||
this.rawPcmListeners.add(listener);
|
||||
return () => this.rawPcmListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns
|
||||
* false) if an STT capture already owns the mic; the caller re-tries when its
|
||||
* eligibility is re-evaluated. Idempotent while already running.
|
||||
*/
|
||||
startRawCapture(options: { communicator: any }): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
if (this.rawActive) return true;
|
||||
if (this.captureHolders.size > 0 || this.started) return false;
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setSaveRecordings(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
this.cloudClient = null;
|
||||
this.rawActive = true;
|
||||
this.started = true;
|
||||
// "cloud" mode decodes LC3 to PCM and emits it via onPcm without loading
|
||||
// any recognizer; with no cloudClient the frames reach only the raw tap.
|
||||
this.controller?.start("cloud");
|
||||
return true;
|
||||
}
|
||||
|
||||
stopRawCapture(): void {
|
||||
if (!this.rawActive) return;
|
||||
this.rawActive = false;
|
||||
this.started = false;
|
||||
if (global.isAndroid) this.controller?.stop();
|
||||
}
|
||||
|
||||
private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void {
|
||||
if (!global.isAndroid) return;
|
||||
// STT preempts the raw-PCM tap: a live transcription session owns the mic,
|
||||
// and an EvenHub app gets no audio for its duration.
|
||||
if (this.rawActive) this.stopRawCapture();
|
||||
const wasIdle = this.captureHolders.size === 0;
|
||||
this.captureHolders.add(holder);
|
||||
if (!wasIdle) {
|
||||
@@ -196,6 +242,7 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
stop(): void {
|
||||
this.captureHolders.clear();
|
||||
this.rawActive = false;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
@@ -214,6 +261,9 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller = new com.faceclaw.app.FaceclawVoiceController(context);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({
|
||||
onStatus: (status: string) => {
|
||||
// The raw tap runs silently; its "Listening (cloud)..." chatter must
|
||||
// not surface on the glasses voice status.
|
||||
if (this.rawActive) return;
|
||||
this.setStatus(String(status));
|
||||
},
|
||||
onWakeWord: (keyword: string) => {
|
||||
@@ -225,7 +275,11 @@ export class FaceclawVoiceControlBridge {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
this.cloudClient?.acceptPcm(toUint8Array(pcm));
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
for (const listener of this.speechEndListeners) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
||||
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
||||
import { MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
|
||||
import { voiceActivity } from "./voice-activity";
|
||||
import { AssistantLayer } from "./assistant";
|
||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
@@ -675,6 +676,7 @@ class Shell {
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === layer) {
|
||||
this.activeVoiceLayer = null;
|
||||
voiceActivity.setActive(false);
|
||||
// The idle countdown restarts in full once voice input ends.
|
||||
this.noteUserActivity();
|
||||
}
|
||||
@@ -689,6 +691,7 @@ class Shell {
|
||||
autoSend,
|
||||
});
|
||||
this.activeVoiceLayer = layer;
|
||||
voiceActivity.setActive(true);
|
||||
this.stack.push(layer);
|
||||
layer.startCapture();
|
||||
}
|
||||
@@ -850,6 +853,7 @@ class Shell {
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === voice) {
|
||||
this.activeVoiceLayer = null;
|
||||
voiceActivity.setActive(false);
|
||||
this.noteUserActivity();
|
||||
}
|
||||
},
|
||||
@@ -864,6 +868,7 @@ class Shell {
|
||||
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
|
||||
});
|
||||
this.activeVoiceLayer = voice;
|
||||
voiceActivity.setActive(true);
|
||||
this.stack.push(voice);
|
||||
voice.startCapture();
|
||||
this.config.requestShellRender();
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* A tiny global signal for whether a voice-input modal (the assistant dialog)
|
||||
* is currently open. The shell sets it; the EvenHub mic router reads it to
|
||||
* decide whether an app may hold the glasses microphone — an app receives no
|
||||
* audio while the assistant is active, since both share the one G2 mic.
|
||||
*
|
||||
* Kept as a dependency-free leaf so both the shell and the EvenHub layer can
|
||||
* import it without a cycle.
|
||||
*/
|
||||
|
||||
type Listener = (active: boolean) => void;
|
||||
|
||||
let active = false;
|
||||
const listeners = new Set<Listener>();
|
||||
|
||||
export const voiceActivity = {
|
||||
isActive(): boolean {
|
||||
return active;
|
||||
},
|
||||
setActive(next: boolean): void {
|
||||
if (active === next) return;
|
||||
active = next;
|
||||
listeners.forEach((listener) => listener(active));
|
||||
},
|
||||
subscribe(listener: Listener): () => void {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user