diff --git a/app/apps/evenhub/ehpk.ts b/app/apps/evenhub/ehpk.ts index 3c1c42de..ba71fc1a 100644 --- a/app/apps/evenhub/ehpk.ts +++ b/app/apps/evenhub/ehpk.ts @@ -10,6 +10,7 @@ * an integrity check only, not a signature; we skip verifying it. */ import { decompress } from "fzstd"; +import { parsePermissions, type EvenHubPermission } from "./permissions"; const XOR_KEY = "EVEN REALITIES"; @@ -31,6 +32,8 @@ export type EvenHubManifest = { entrypoint: string; /** Hosts the app declares it will reach; not yet enforced. */ networkWhitelist: string[]; + /** Normalized declared permissions (both manifest shapes). */ + permissions: EvenHubPermission[]; raw: Record; }; @@ -115,19 +118,13 @@ export function parseManifest(appJsonText: string): EvenHubManifest { const version = String(raw.version ?? "0"); const entrypoint = String(raw.entrypoint ?? "index.html"); + const permissions = parsePermissions(raw.permissions); const networkWhitelist: string[] = []; - const permissions = raw.permissions; - if (Array.isArray(permissions)) { - for (const perm of permissions) { - if (perm && typeof perm === "object" && (perm as Record).name === "network") { - const whitelist = (perm as Record).whitelist; - if (Array.isArray(whitelist)) networkWhitelist.push(...whitelist.map(String)); - } + for (const permission of permissions) { + if (permission.name === "network" && permission.whitelist) { + networkWhitelist.push(...permission.whitelist); } - } else if (permissions && typeof permissions === "object") { - const network = (permissions as Record).network; - if (Array.isArray(network)) networkWhitelist.push(...network.map(String)); } - return { packageId, name, version, entrypoint, networkWhitelist, raw }; + return { packageId, name, version, entrypoint, networkWhitelist, permissions, raw }; } diff --git a/app/apps/evenhub/evenhub-window.ts b/app/apps/evenhub/evenhub-window.ts index 86c9e10a..f25a2a06 100644 --- a/app/apps/evenhub/evenhub-window.ts +++ b/app/apps/evenhub/evenhub-window.ts @@ -77,5 +77,12 @@ export function createEvenHubWindow( session.setForeground(foreground); }; + // Screen on/off gates the app's microphone (silent while the screen is off). + const baseSetScreenOn = created.window.setScreenOn; + created.window.setScreenOn = (on) => { + baseSetScreenOn?.(on); + session.setScreenOn(on); + }; + return created; } diff --git a/app/apps/evenhub/installed-apps.ts b/app/apps/evenhub/installed-apps.ts index 7065aae2..087dd113 100644 --- a/app/apps/evenhub/installed-apps.ts +++ b/app/apps/evenhub/installed-apps.ts @@ -142,6 +142,21 @@ export function installEvenHubPackageFile(path: string): InstalledEvenHubApp { return installEvenHubPackageBytes(bytes); } +/** + * Read an .ehpk's manifest without installing it — used by the Files app to + * show the permission-confirmation dialog before install/run. Returns null if + * the file can't be read or parsed. + */ +export function readEvenHubPackageManifest(path: string): EvenHubManifest | null { + const bytes = readBinaryFile(path); + if (!bytes) return null; + try { + return inspectEhpk(bytes).manifest; + } catch { + return null; + } +} + /** Remove the stored EHPK, unpacked runtime files, and registry entry. */ export function uninstallEvenHubPackage(packageId: string): boolean { const apps = getInstalledEvenHubApps(); diff --git a/app/apps/evenhub/mic-router.ts b/app/apps/evenhub/mic-router.ts new file mode 100644 index 00000000..fca3e374 --- /dev/null +++ b/app/apps/evenhub/mic-router.ts @@ -0,0 +1,93 @@ +/** + * Arbitrates the single G2 microphone among running EvenHub apps. + * + * An app asks for the mic with audioControl(true); it only actually receives + * audio while it is the foreground (visible) window, the screen is on, and no + * voice-input modal (the assistant) is active. When any of those stops holding, + * the app simply stops receiving frames — it is not told, matching the stock + * contract where a backgrounded app's mic goes silent. + * + * Because only one window is foreground at a time, at most one requesting app + * is ever eligible. The router runs the shared decode-only raw-PCM tap + * (voiceControlBridge) while an eligible app exists and routes each decoded + * frame to it; STT sessions preempt the tap at the bridge level. + */ +import { voiceControlBridge } from "../../native/voice-control"; +import { voiceActivity } from "../../ui/shell/voice-activity"; + +declare const com: any; + +export type EvenHubMicClient = { + readonly windowId: string; + isForeground(): boolean; + isScreenOn(): boolean; + deliverAudioPcm(pcm: Uint8Array): void; +}; + +function activeCommunicator(): any { + if (!global.isAndroid) return null; + try { + return com.faceclaw.app.FaceclawBleCommunicator.getActive(); + } catch { + return null; + } +} + +class EvenHubMicRouter { + private readonly requesting = new Set(); + private active: EvenHubMicClient | null = null; + private unsubscribeRawPcm: (() => void) | null = null; + private unsubscribeVoiceActivity: (() => void) | null = null; + + /** An app enabled its microphone (audioControl true). */ + requestMic(client: EvenHubMicClient): void { + this.requesting.add(client); + this.ensureSubscriptions(); + this.evaluate(); + } + + /** An app disabled its microphone, backgrounded, or closed. */ + releaseMic(client: EvenHubMicClient): void { + if (!this.requesting.delete(client)) return; + this.evaluate(); + } + + /** A requesting app's foreground/screen state changed. */ + notifyEligibilityChanged(): void { + this.evaluate(); + } + + private ensureSubscriptions(): void { + if (!this.unsubscribeRawPcm) { + this.unsubscribeRawPcm = voiceControlBridge.onRawPcm((pcm) => { + this.active?.deliverAudioPcm(pcm); + }); + } + if (!this.unsubscribeVoiceActivity) { + this.unsubscribeVoiceActivity = voiceActivity.subscribe(() => this.evaluate()); + } + } + + private eligible(): EvenHubMicClient | null { + if (voiceActivity.isActive()) return null; + return Array.from(this.requesting).find( + (client) => client.isForeground() && client.isScreenOn(), + ) ?? null; + } + + private evaluate(): void { + const next = this.eligible(); + this.active = next; + if (next) { + // Idempotent: no-op if already running, and re-asserted on every + // eligibility change so it self-heals after an STT preemption. Returns + // false if STT currently owns the mic; a later trigger (assistant modal + // close, foreground/screen change) re-tries. + voiceControlBridge.startRawCapture({ communicator: activeCommunicator() }); + } else { + voiceControlBridge.stopRawCapture(); + } + } +} + +export const evenHubMicRouter = new EvenHubMicRouter(); diff --git a/app/apps/evenhub/permission-dialog.ts b/app/apps/evenhub/permission-dialog.ts new file mode 100644 index 00000000..2211358d --- /dev/null +++ b/app/apps/evenhub/permission-dialog.ts @@ -0,0 +1,117 @@ +import { getDefaultSmallFont } from "../../graphics/bdffont"; +import { GrayImage } from "../../graphics/image"; +import { truncateText } from "../../graphics/textwrap"; +import { Layer, type DashboardInputEvent, type LayerContext, type PaintBelow } from "../../ui/layers"; +import { drawSelectionHighlight } from "../../ui/menu"; +import { permissionDetail, permissionLabel, type EvenHubPermission } from "./permissions"; + +const DIALOG_X = 8; +const DIALOG_Y = 8; +const DIALOG_WIDTH = 272; +const PADDING = 10; +const HEADER_STEP = 16; +const PERM_LABEL_STEP = 15; +const PERM_DETAIL_STEP = 13; +const PERM_GAP = 4; +const ACTION_ROW_HEIGHT = 20; + +/** + * Confirmation dialog listing the permissions an EvenHub app declares, shown + * before installing it or running an uninstalled package. Allow proceeds; + * Cancel (or double-click) backs out. Apps that declare no permissions never + * reach this dialog — the caller runs them directly. + */ +export class EvenHubPermissionDialogLayer implements Layer { + /** 0 = Allow, 1 = Cancel. */ + private selectedIndex = 0; + private resolved = false; + + constructor( + private readonly appName: string, + private readonly permissions: EvenHubPermission[], + private readonly onConfirm: () => void, + private readonly onCancel: () => void, + ) {} + + paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { + const font = getDefaultSmallFont(); + const { height: viewportHeight } = ctx.stack.getBaseSize(); + const image = paintBelow(); + const textWidth = DIALOG_WIDTH - 2 * PADDING - 4; + const left = DIALOG_X + PADDING + 2; + + const permsHeight = this.permissions.reduce( + (sum, permission) => sum + PERM_LABEL_STEP + (permissionDetail(permission) ? PERM_DETAIL_STEP : 0) + PERM_GAP, + 0, + ); + const bodyTop = PADDING + HEADER_STEP + 6 + permsHeight + 6; + const height = Math.min(bodyTop + 2 * ACTION_ROW_HEIGHT + PADDING, viewportHeight - 2 * DIALOG_Y); + + // Fill 1 (transparent color key is 0), outline for the dialog edge. + image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 1); + image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 72); + + let y = DIALOG_Y + PADDING; + image.drawText(font, left, y, truncateText(font, `${this.appName} needs:`, textWidth), 235); + y += HEADER_STEP + 6; + + for (const permission of this.permissions) { + image.drawText(font, left, y, truncateText(font, permissionLabel(permission.name), textWidth), 220); + y += PERM_LABEL_STEP; + const detail = permissionDetail(permission); + if (detail) { + image.drawText(font, left + 8, y, truncateText(font, detail, textWidth - 8), 140); + y += PERM_DETAIL_STEP; + } + y += PERM_GAP; + } + y += 2; + + const focused = ctx.stack.isFocused(); + const actions = ["Allow", "Cancel"]; + for (let index = 0; index < actions.length; index++) { + const rowY = y + index * ACTION_ROW_HEIGHT; + const selected = index === this.selectedIndex; + if (selected) { + drawSelectionHighlight(image, DIALOG_X + 12, rowY, DIALOG_WIDTH - 24, ACTION_ROW_HEIGHT - 1, focused, 8); + } + image.drawText(font, DIALOG_X + 22, rowY + 3, actions[index]!, selected ? 255 : 200); + } + return image; + } + + handleInput(event: DashboardInputEvent, ctx: LayerContext): void { + switch (event.type) { + case "scroll-up": + this.selectedIndex = (this.selectedIndex + 1) % 2; + return; + case "scroll-down": + this.selectedIndex = (this.selectedIndex + 1) % 2; + return; + case "click": + this.resolve(ctx, this.selectedIndex === 0); + return; + case "double-click": + this.resolve(ctx, false); + return; + default: + return; + } + } + + onRemoved(): void { + // Dismissed some other way (window closed, back button): treat as cancel. + if (!this.resolved) { + this.resolved = true; + this.onCancel(); + } + } + + private resolve(ctx: LayerContext, confirmed: boolean): void { + if (this.resolved) return; + this.resolved = true; + ctx.stack.pop(); + if (confirmed) this.onConfirm(); + else this.onCancel(); + } +} diff --git a/app/apps/evenhub/permissions.ts b/app/apps/evenhub/permissions.ts new file mode 100644 index 00000000..2a1464d2 --- /dev/null +++ b/app/apps/evenhub/permissions.ts @@ -0,0 +1,155 @@ +/** + * EvenHub app permissions: parsing the manifest's declared permissions and + * turning them into the human-readable list Faceclaw shows for confirmation. + * + * Two manifest shapes exist in the wild (see notes/evenhub_compatibility.txt): + * - Current SDK: `permissions` is an array of {name, desc?, whitelist?}, with + * names network, location, g2-microphone, phone-microphone, album, camera. + * - Older shape: `permissions` is a map {"network": [...hosts], "fs": [...]}. + * + * Permissions are confirmed by the user when an app is installed, or when an + * uninstalled .ehpk is run directly (installing implies a prior confirmation). + * Once past that gate the running app is treated as having been granted every + * permission it declared; runtime surfaces (e.g. the microphone) check the + * declared set to decide whether to honor a request. + */ + +export type EvenHubPermissionName = + | "network" + | "location" + | "g2-microphone" + | "phone-microphone" + | "album" + | "camera" + | "fs" + | string; + +export type EvenHubPermission = { + name: EvenHubPermissionName; + /** App-supplied description, when the manifest provides one. */ + description?: string; + /** For network/fs: the declared host or path whitelist. */ + whitelist?: string[]; +}; + +/** The mic permission names an audioControl request may be gated behind. */ +const MICROPHONE_PERMISSIONS = new Set(["g2-microphone", "phone-microphone", "microphone"]); + +/** Short label shown as the permission's title in the confirmation dialog. */ +export function permissionLabel(name: EvenHubPermissionName): string { + switch (name) { + case "network": + return "Network access"; + case "location": + return "Location"; + case "g2-microphone": + return "Glasses microphone"; + case "phone-microphone": + return "Phone microphone"; + case "microphone": + return "Microphone"; + case "album": + return "Photo album"; + case "camera": + return "Camera"; + case "fs": + return "Bundled files"; + default: + return name; + } +} + +/** Secondary line under the label: the app's own description, or a default. */ +export function permissionDetail(permission: EvenHubPermission): string { + if (permission.description && permission.description.trim()) { + return permission.description.trim(); + } + if ((permission.name === "network" || permission.name === "fs") && permission.whitelist?.length) { + return permission.whitelist.join(", "); + } + switch (permission.name) { + case "network": + return "Reach servers on the internet"; + case "location": + return "Read your location"; + case "g2-microphone": + return "Listen through the glasses microphone"; + case "phone-microphone": + return "Listen through the phone microphone"; + case "microphone": + return "Listen through the microphone"; + case "album": + return "Read photos from your album"; + case "camera": + return "Capture photos from the camera"; + default: + return ""; + } +} + +export function isMicrophonePermission(name: EvenHubPermissionName): boolean { + return MICROPHONE_PERMISSIONS.has(name); +} + +/** Whether the declared set includes any microphone permission. */ +export function permissionsIncludeMicrophone(permissions: EvenHubPermission[]): boolean { + return permissions.some((permission) => isMicrophonePermission(permission.name)); +} + +/** + * Parse the manifest's raw `permissions` value into a normalized list, + * accepting both the array and the older map shape. Unknown names pass through + * so the confirmation dialog still surfaces them. + */ +export function parsePermissions(raw: unknown): EvenHubPermission[] { + const out: EvenHubPermission[] = []; + const seen = new Set(); + const add = (permission: EvenHubPermission) => { + const name = String(permission.name).trim(); + if (!name || seen.has(name)) return; + seen.add(name); + out.push({ ...permission, name }); + }; + + if (Array.isArray(raw)) { + for (const entry of raw) { + if (typeof entry === "string") { + add({ name: entry }); + } else if (entry && typeof entry === "object") { + const record = entry as Record; + const name = record.name; + if (typeof name !== "string" || !name.trim()) continue; + const description = typeof record.desc === "string" + ? record.desc + : typeof record.description === "string" + ? record.description + : undefined; + const whitelist = Array.isArray(record.whitelist) + ? record.whitelist.map(String) + : undefined; + add({ name, description, whitelist }); + } + } + } else if (raw && typeof raw === "object") { + // Older map shape: {"network": ["host", ...], "fs": ["./assets"]}. + for (const [name, value] of Object.entries(raw as Record)) { + const whitelist = Array.isArray(value) ? value.map(String) : undefined; + add({ name, whitelist }); + } + } + return out; +} + +/** + * A stable signature of a granted permission set, for comparing what the user + * confirmed against what a later version of the app declares. + */ +export function permissionsSignature(permissions: EvenHubPermission[]): string { + return permissions + .map((permission) => { + const list = permission.whitelist?.length ? `(${[...permission.whitelist].sort().join(",")})` : ""; + return `${permission.name}${list}`; + }) + .sort() + .join(";"); +} diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index b1c28e86..dc6da845 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -34,6 +34,8 @@ import { } from "./containers"; import { compositePage } from "./compositor"; import { type EvenHubManifest } from "./ehpk"; +import { permissionsIncludeMicrophone } from "./permissions"; +import { evenHubMicRouter, type EvenHubMicClient } from "./mic-router"; const UPNG = require("upng-js"); @@ -166,7 +168,7 @@ export type EvenHubWindowHooks = { focusSwitcher: () => void; }; -export class EvenHubSession { +export class EvenHubSession implements EvenHubMicClient { readonly manifest: EvenHubManifest; readonly distDir: string; @@ -179,6 +181,10 @@ export class EvenHubSession { private systemExitSent = false; /** Shell focus state; true from launch (apps open focused). */ private foreground = true; + /** Phone screen state; the mic goes silent while the screen is off. */ + private screenOn = true; + /** The app has enabled its microphone via audioControl(true). */ + private micRequested = false; private log: (message: string) => void; constructor(manifest: EvenHubManifest, distDir: string, log: (message: string) => void) { @@ -288,10 +294,66 @@ export class EvenHubSession { setForeground(foreground: boolean): void { if (this.foreground === foreground) return; this.foreground = foreground; + // Foreground gates mic eligibility (an app only hears audio while visible). + if (this.micRequested) evenHubMicRouter.notifyEligibilityChanged(); if (!this.pageCreated) return; this.emitSysEvent(foreground ? FOREGROUND_ENTER_EVENT : FOREGROUND_EXIT_EVENT, 0); } + /** Screen turned on/off (forwarded from the shell); mutes the mic when off. */ + setScreenOn(on: boolean): void { + if (this.screenOn === on) return; + this.screenOn = on; + if (this.micRequested) evenHubMicRouter.notifyEligibilityChanged(); + } + + // ----- microphone (EvenHubMicClient) ----- + + get windowId(): string { + return this.manifest.packageId; + } + + isForeground(): boolean { + return this.foreground; + } + + isScreenOn(): boolean { + return this.screenOn; + } + + /** A decoded mic frame from the router: hand it to the app as an audioEvent. */ + deliverAudioPcm(pcm: Uint8Array): void { + const audioPcm = Array.from(pcm); + this.pushMessage("evenHubEvent", { type: "audioEvent", jsonData: { audioPcm } }); + } + + /** + * audioControl(enable, source?): open or close the microphone for this app. + * Gated on a declared microphone permission (confirmed at install/first run). + * Enabling only registers intent — the router decides when audio actually + * flows (foreground + screen on + no assistant modal). + */ + private audioControl(data: Record): boolean { + if (!permissionsIncludeMicrophone(this.manifest.permissions)) { + this.log("evenhub: audioControl denied (app declares no microphone permission)"); + return false; + } + const enable = readAudioEnable(data); + // The 0.0.12 payload shape isn't captured; log it once per toggle so the + // real field names can be confirmed from hardware. + this.log(`evenhub: audioControl enable=${enable} data=${JSON.stringify(data)}`); + if (enable) { + if (!this.micRequested) { + this.micRequested = true; + evenHubMicRouter.requestMic(this); + } + } else if (this.micRequested) { + this.micRequested = false; + evenHubMicRouter.releaseMic(this); + } + return true; + } + // ----- bridge calls (web -> host) ----- /** @@ -351,8 +413,9 @@ export class EvenHubSession { isInCase: false, }, }; - // Unsupported surface (mic, IMU, location, pickers): fail politely. case "audioControl": + return this.audioControl(data); + // Unsupported surface (IMU, location, pickers): fail politely. case "imuControl": case "startAppLocationUpdates": case "stopAppLocationUpdates": @@ -497,6 +560,10 @@ export class EvenHubSession { close(): void { if (this.closed) return; this.closed = true; + if (this.micRequested) { + this.micRequested = false; + evenHubMicRouter.releaseMic(this); + } if (!this.systemExitSent) { // Best effort: lets the app's teardown handlers run before the webview dies. this.emitSysEvent(SYSTEM_EXIT_EVENT, 0); @@ -522,6 +589,22 @@ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Read the enable flag from an audioControl payload. The exact wire field name + * for SDK 0.0.12 isn't captured, so accept the plausible spellings; default to + * enabling (the common intent) when the payload is opaque, and log it so the + * real shape can be pinned from hardware. + */ +function readAudioEnable(data: Record): boolean { + for (const key of ["enable", "open", "isOpen", "on", "status", "value", "start"]) { + const value = data[key]; + if (typeof value === "boolean") return value; + if (typeof value === "number") return value !== 0; + if (typeof value === "string") return value === "true" || value === "1" || value === "on" || value === "open"; + } + return true; +} + /** ring/left-arm/right-arm -> PB EventSourceType. */ function gestureSource(source: "ring" | "left-arm" | "right-arm"): number { switch (source) { diff --git a/app/apps/files/files-app.ts b/app/apps/files/files-app.ts index 33343f21..12eab782 100644 --- a/app/apps/files/files-app.ts +++ b/app/apps/files/files-app.ts @@ -1,8 +1,11 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access"; import { isFontFile } from "../../native/font-files"; import { isDecodableImageFile } from "../../native/image-files"; +import { readEvenHubPackageManifest } from "../evenhub/installed-apps"; +import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog"; import { FileBrowserLayer } from "./file-browser"; import { FileInfoDialogLayer, type FileInfoAction } from "./file-info-dialog"; +import { type LayerContext } from "../../ui/layers"; import { FontPreviewerLayer } from "./font-previewer"; import { ImageViewerLayer } from "./image-viewer"; import { TextViewerLayer } from "./text-viewer"; @@ -138,6 +141,22 @@ export function createImageDocumentWindow( }); } +/** + * Show the permission-confirmation dialog for an .ehpk before install/run, + * then run `proceed` if the user allows. Packages that declare no permissions + * skip the dialog and proceed immediately. + */ +function confirmEhpkPermissions(ctx: LayerContext, entry: DirectoryEntry, proceed: () => void): void { + const manifest = readEvenHubPackageManifest(entry.path); + if (!manifest || manifest.permissions.length === 0) { + proceed(); + return; + } + ctx.stack.push( + new EvenHubPermissionDialogLayer(manifest.name, manifest.permissions, proceed, () => {}), + ); +} + /** * The open actions for the picked-file dialog: View here / Open in new * window for viewable types, empty for everything else (metadata only). @@ -169,14 +188,16 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI label: "Run app", onSelect: (ctx) => { ctx.stack.pop(); - options.openEhpkApp(entry.path); + confirmEhpkPermissions(ctx, entry, () => options.openEhpkApp(entry.path)); }, }, { label: "Install", - onSelect: async (ctx) => { + onSelect: (ctx) => { ctx.stack.pop(); - await options.installEhpkApp(entry.path); + confirmEhpkPermissions(ctx, entry, () => { + void options.installEhpkApp(entry.path); + }); }, }, ]; diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index da5f10c9..a5686680 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -41,6 +41,8 @@ export type PushToTalkOptions = { /** Who currently wants the mic running. */ type CaptureHolder = "ptt" | "continuous"; +export type RawPcmListener = (pcm: Uint8Array) => void; + export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly wakeWordListeners = new Set<(keyword: string) => void>(); @@ -57,6 +59,12 @@ export class FaceclawVoiceControlBridge { private readonly captureHolders = new Set(); // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: CloudSttClient | null = null; + // Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the + // decode-only "cloud" mode and every decoded PCM frame is broadcast to these + // listeners. STT capture preempts it — a live assistant/transcribe session + // owns the single G2 mic and the raw tap is stopped for its duration. + private readonly rawPcmListeners = new Set(); + private rawActive = false; onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); @@ -102,8 +110,46 @@ export class FaceclawVoiceControlBridge { this.releaseCapture("continuous", false); } + /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ + onRawPcm(listener: RawPcmListener): () => void { + this.rawPcmListeners.add(listener); + return () => this.rawPcmListeners.delete(listener); + } + + /** + * Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns + * false) if an STT capture already owns the mic; the caller re-tries when its + * eligibility is re-evaluated. Idempotent while already running. + */ + startRawCapture(options: { communicator: any }): boolean { + if (!global.isAndroid) return false; + if (this.rawActive) return true; + if (this.captureHolders.size > 0 || this.started) return false; + this.ensureController(); + this.controller?.setCommunicator(options.communicator); + this.controller?.setSaveRecordings(false); + this.controller?.setEndpointing(false); + this.cloudClient = null; + this.rawActive = true; + this.started = true; + // "cloud" mode decodes LC3 to PCM and emits it via onPcm without loading + // any recognizer; with no cloudClient the frames reach only the raw tap. + this.controller?.start("cloud"); + return true; + } + + stopRawCapture(): void { + if (!this.rawActive) return; + this.rawActive = false; + this.started = false; + if (global.isAndroid) this.controller?.stop(); + } + private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void { if (!global.isAndroid) return; + // STT preempts the raw-PCM tap: a live transcription session owns the mic, + // and an EvenHub app gets no audio for its duration. + if (this.rawActive) this.stopRawCapture(); const wasIdle = this.captureHolders.size === 0; this.captureHolders.add(holder); if (!wasIdle) { @@ -196,6 +242,7 @@ export class FaceclawVoiceControlBridge { stop(): void { this.captureHolders.clear(); + this.rawActive = false; if (global.isAndroid) { this.controller?.stop(); } @@ -214,6 +261,9 @@ export class FaceclawVoiceControlBridge { this.controller = new com.faceclaw.app.FaceclawVoiceController(context); this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({ onStatus: (status: string) => { + // The raw tap runs silently; its "Listening (cloud)..." chatter must + // not surface on the glasses voice status. + if (this.rawActive) return; this.setStatus(String(status)); }, onWakeWord: (keyword: string) => { @@ -225,7 +275,11 @@ export class FaceclawVoiceControlBridge { this.emitTranscript(String(text), Boolean(isFinal)); }, onPcm: (pcm: any) => { - this.cloudClient?.acceptPcm(toUint8Array(pcm)); + const bytes = toUint8Array(pcm); + this.cloudClient?.acceptPcm(bytes); + if (this.rawPcmListeners.size > 0) { + this.rawPcmListeners.forEach((listener) => listener(bytes)); + } }, onSpeechEnd: () => { for (const listener of this.speechEndListeners) { diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index d9945ec1..27554750 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -5,6 +5,7 @@ import type { RawInputEvent } from "../../native/faceclaw-communicator"; import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers"; import { MenuLayer, type MenuItem } from "../menu"; import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input"; +import { voiceActivity } from "./voice-activity"; import { AssistantLayer } from "./assistant"; import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session"; import { resolveAssistantModel } from "../../assistant/models"; @@ -675,6 +676,7 @@ class Shell { onClosed: () => { if (this.activeVoiceLayer === layer) { this.activeVoiceLayer = null; + voiceActivity.setActive(false); // The idle countdown restarts in full once voice input ends. this.noteUserActivity(); } @@ -689,6 +691,7 @@ class Shell { autoSend, }); this.activeVoiceLayer = layer; + voiceActivity.setActive(true); this.stack.push(layer); layer.startCapture(); } @@ -850,6 +853,7 @@ class Shell { onClosed: () => { if (this.activeVoiceLayer === voice) { this.activeVoiceLayer = null; + voiceActivity.setActive(false); this.noteUserActivity(); } }, @@ -864,6 +868,7 @@ class Shell { autoSend: handsFree && assistantSkipConfirmationSetting.get(), }); this.activeVoiceLayer = voice; + voiceActivity.setActive(true); this.stack.push(voice); voice.startCapture(); this.config.requestShellRender(); diff --git a/app/ui/shell/voice-activity.ts b/app/ui/shell/voice-activity.ts new file mode 100644 index 00000000..e38acf82 --- /dev/null +++ b/app/ui/shell/voice-activity.ts @@ -0,0 +1,29 @@ +/** + * A tiny global signal for whether a voice-input modal (the assistant dialog) + * is currently open. The shell sets it; the EvenHub mic router reads it to + * decide whether an app may hold the glasses microphone — an app receives no + * audio while the assistant is active, since both share the one G2 mic. + * + * Kept as a dependency-free leaf so both the shell and the EvenHub layer can + * import it without a cycle. + */ + +type Listener = (active: boolean) => void; + +let active = false; +const listeners = new Set(); + +export const voiceActivity = { + isActive(): boolean { + return active; + }, + setActive(next: boolean): void { + if (active === next) return; + active = next; + listeners.forEach((listener) => listener(active)); + }, + subscribe(listener: Listener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); + }, +};