Add support for buzzer sound effects on iOS

This commit is contained in:
James Babcock committed 2026-09-19 12:03:57 -07:00
1 parent 1412db8e82
commit e50da605e0
19 files changed
+211 -48

No files matched your search

+4 -7
View File
@@ -17,6 +17,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
import { getFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import * as frameTimings from "../../native/frame-timings";
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
import { getActiveDisplay } from "../../native/active-display";
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
@@ -33,7 +34,6 @@ import {
} from "../../ui/gestures";
declare const global: any;
declare const com: any;
const largeFont = getFont("terminus32");
const mediumFont = getFont("terminus24");
@@ -295,16 +295,13 @@ function inferForeground(window: BlocksWindow, focused: boolean): void {
/**
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
* steps on its own timer, and the Java call is safe from the worker thread
* (same path as frame submission). Effects never exceed one message, so no
* phrase pacing is needed.
* steps on its own timer; the platform bridge routes it from the worker.
* Effects never exceed one message, so no phrase pacing is needed.
*/
function playSfx(window: BlocksWindow, steps: Step[]): void {
if (!window.soundOn || steps.length === 0) return;
try {
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
if (!communicator) return;
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
} catch (error) {
console.warn(`blocks sfx failed: ${error}`);
}
+2 -1
View File
@@ -103,7 +103,8 @@ async function startApp(
cleanup?: () => void,
): Promise<void> {
const windowId = `evenhub:app:${nextSerial++}`;
const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl);
const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl,
payload => ctx.actions.playBuzzerSequence(payload));
const webView = createEvenHubWebView(session);
session.attachWebView({
evaluateJs: webView.evaluateJs,
+2 -14
View File
@@ -65,8 +65,6 @@ import { ensureFineLocationPermission } from "../../native/location-permissions"
const UPNG = require("upng-js");
declare const com: any;
/** OsEventTypeList values (PB ordinals). */
const CLICK_EVENT = 0;
const SCROLL_TOP_EVENT = 1;
@@ -371,6 +369,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
distDir: string,
log: (message: string) => void,
remoteUrl = "",
private readonly sendBuzzerSequence: (payload: Uint8Array) => Promise<void> | void = () => {},
) {
this.manifest = manifest;
this.distDir = distDir;
@@ -1164,13 +1163,11 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
private async playBuzzer(rawSteps: unknown[]): Promise<void> {
const steps = normalizeBuzzerSteps(rawSteps);
if (steps.length === 0) return;
const communicator = activeCommunicator();
if (!communicator) return;
for (let i = 0; i < steps.length; i += CFW_SEQ_MAX) {
const chunk = steps.slice(i, i + CFW_SEQ_MAX);
const payload = buildSoundSequencePayload(chunk);
try {
communicator.playBuzzerSequence(payload.buffer);
await this.sendBuzzerSequence(payload);
} catch (error) {
this.log(`evenhub: playBuzzer failed: ${error}`);
return;
@@ -1274,15 +1271,6 @@ function clampImuFreq(freq: number): number {
return Math.min(1000, Math.max(100, snapped));
}
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/**
* Copy content into a `preserve` container from the same-named, same-kind
* container in the previous page (if any), so a replaceLayout keeps its pixels /
+3 -5
View File
@@ -31,6 +31,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
import { getFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import * as frameTimings from "../../native/frame-timings";
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
import { getActiveDisplay } from "../../native/active-display";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
@@ -64,7 +65,6 @@ import {
} from "./flappy-sprites";
declare const global: any;
declare const com: any;
const largeFont = getFont("terminus32");
const mediumFont = getFont("terminus24");
@@ -311,7 +311,7 @@ function saveHighScore(window: FlappyWindow): void {
/**
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
* steps on its own timer, and the Java call is safe from the worker thread.
* steps on its own timer; the platform bridge routes it from the worker.
* Minor (frequent) effects are dropped when they'd arrive within the
* rate-limit gap; newsworthy ones always play.
*/
@@ -321,9 +321,7 @@ function playSfx(window: FlappyWindow, steps: Step[], minor = false): void {
if (minor && now - window.lastMinorSfxAtMs < MINOR_SFX_MIN_GAP_MS) return;
if (minor) window.lastMinorSfxAtMs = now;
try {
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
if (!communicator) return;
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
} catch (error) {
console.warn(`flappy sfx failed: ${error}`);
}
+3 -5
View File
@@ -25,6 +25,7 @@ import { getFont } from "../../graphics/bdffont";
import { ensurePreinstalledFonts, installedFontPath } from "../../graphics/installed-fonts";
import { TtfFont } from "../../graphics/ttf-font";
import * as frameTimings from "../../native/frame-timings";
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
import { getActiveDisplay } from "../../native/active-display";
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
@@ -34,7 +35,6 @@ import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-win
import { directionalFallback, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, type InputEvent } from "../../ui/gestures";
declare const global: any;
declare const com: any;
ensurePreinstalledFonts();
const fontPath = installedFontPath("Roboto-Regular.ttf");
@@ -206,14 +206,12 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
/**
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
* steps on its own timer, and the Java call is safe from the worker thread.
* steps on its own timer; the platform bridge routes it from the worker.
*/
function playSfx(window: FreecellWindow, steps: Step[]): void {
if (!window.soundOn || steps.length === 0) return;
try {
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
if (!communicator) return;
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
} catch (error) {
console.warn(`freecell sfx failed: ${error}`);
}
@@ -25,6 +25,7 @@ import { getFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import * as frameTimings from "../../native/frame-timings";
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
import { getActiveDisplay } from "../../native/active-display";
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
@@ -43,7 +44,6 @@ import {
import { clamp } from "../../util/numeric-util";
declare const global: any;
declare const com: any;
const largeFont = getFont("terminus32");
const mediumFont = getFont("terminus24");
@@ -278,14 +278,12 @@ function saveDifficultyIndex(window: MinesweeperWindow): void {
/**
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
* steps on its own timer, and the Java call is safe from the worker thread.
* steps on its own timer; the platform bridge routes it from the worker.
*/
function playSfx(window: MinesweeperWindow, steps: Step[]): void {
if (!window.soundOn || steps.length === 0) return;
try {
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
if (!communicator) return;
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
} catch (error) {
console.warn(`minesweeper sfx failed: ${error}`);
}
+3 -5
View File
@@ -27,6 +27,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
import { getFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import * as frameTimings from "../../native/frame-timings";
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
import { getActiveDisplay } from "../../native/active-display";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
@@ -44,7 +45,6 @@ import {
import { clamp } from "../../util/numeric-util";
declare const global: any;
declare const com: any;
const largeFont = getFont("terminus32");
const mediumFont = getFont("terminus24");
@@ -429,7 +429,7 @@ function saveHighScore(window: PinballWindow): void {
/**
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
* steps on its own timer, and the Java call is safe from the worker thread.
* steps on its own timer; the platform bridge routes it from the worker.
* Minor (frequent) effects are dropped when they'd arrive within the
* rate-limit gap; newsworthy ones always play.
*/
@@ -439,9 +439,7 @@ function playSfx(window: PinballWindow, steps: Step[], minor = false): void {
if (minor && now - window.lastMinorSfxAtMs < MINOR_SFX_MIN_GAP_MS) return;
if (minor) window.lastMinorSfxAtMs = now;
try {
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
if (!communicator) return;
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
} catch (error) {
console.warn(`pinball sfx failed: ${error}`);
}
+7
View File
@@ -427,6 +427,13 @@ export class GlassesSession {
if (this.state.phase !== 'connected') return
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
}
/** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
if (this.state.phase !== 'connected' || !this.layoutCreated || payload.length < 3) return
const generation = this.generation
try { await this.requestCfw('left', new Uint8Array(payload)) }
catch (error) { if (generation === this.generation) this.fail(error, true) }
}
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
const token = ++this.microphoneToken, generation = this.generation
+17
View File
@@ -43,6 +43,8 @@ import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting } from '../ui/dashboard-settings'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state'
import { findSoundEffect, playSoundEffect } from '../ui/sound-effects'
/** iOS host for the shared app registry, shell, compositor and BLE session. */
export class IosPreviewController {
@@ -96,10 +98,12 @@ export class IosPreviewController {
private glassesLocked = false
private lockEnabled = lockScreenEnabledSetting.get()
private session: GlassesSession | null = null
private acknowledgedFrames = 0
private logLines: string[] = []
private logTimer: ReturnType<typeof setTimeout> | null = null
private readonly actions: LayerActions = {
...noopLayerActions,
playBuzzerSequence: payload => this.session?.playBuzzerSequence(payload),
requestRender: () => this.requestShellRender(),
disconnect: () => this.disconnect(),
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
@@ -457,6 +461,7 @@ export class IosPreviewController {
? new IosNavigationSensors(event => worker.postMessage({ type: 'navigation-sensors', event })) : undefined
const host = new WorkerAppHost({
appId, worker, navigationSensors,
playBuzzerSequence: payload => this.actions.playBuzzerSequence(payload),
openUrl: url => { void Utils.openUrl(url) },
configureSurface: async (id, visible, heightMode) => {
this.compositor.configureSurface(id, { ...appViewportRect(heightMode, appId), zOrder: 0, transparency: 'opaque' })
@@ -492,6 +497,7 @@ export class IosPreviewController {
if (error) { this.onError(error); return }
if (!this.session) {
this.session = new GlassesSession(iosBluetooth(), state => {
this.maybePlayWelcomeSound(state)
if (state.phase !== 'connected' && state.phase !== 'connecting') this.glassesWorn = null
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
@@ -517,6 +523,17 @@ export class IosPreviewController {
await this.session.start(addresses)
}
async disconnect(): Promise<void> { await this.session?.stop() }
private maybePlayWelcomeSound(state: SessionState): void {
const firstNewFrame = state.frames > this.acknowledgedFrames
this.acknowledgedFrames = state.frames
// Match Android: wait for an acknowledged frame before consuming the jingle.
if (!firstNewFrame || state.phase !== 'connected' || !isWelcomeSoundPending()) return
setWelcomeSoundPending(false)
const effect = findSoundEffect('questcomplete')
if (effect) void playSoundEffect(effect, payload => this.actions.playBuzzerSequence(payload),
ms => new Promise(resolve => setTimeout(resolve, ms)))
.catch(error => this.logBluetooth(`Welcome sound failed: ${error}`))
}
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
private async prepareVoiceCapture(): Promise<boolean> {
if (this.glassesLocked) return false
+2 -3
View File
@@ -7,9 +7,8 @@ declare const com: any;
* through Java statics because those are shared across isolates, unlike any
* JS-side state.
*
* Frame submission only. Glasses-hardware calls (e.g. playBuzzerSequence)
* should keep using FaceclawBleCommunicator.getActive() directly, so they
* skip cleanly when nothing is connected.
* Frame submission only. Buzzer effects use worker-buzzer, which routes to
* the platform BLE session and skips playback when disconnected.
*/
export function getActiveDisplay(): any {
return (
+12
View File
@@ -0,0 +1,12 @@
import type { WorkerAppReply } from "../ui/shell/worker-window";
declare const com: any;
/** Workers use the main-thread BLE session on iOS and the native queue on Android. */
export function playWorkerBuzzerSequence(payload: Uint8Array): void {
if (global.isIOS) {
global.postMessage({ type: "buzzer-sequence", payload: Array.from(payload) } satisfies WorkerAppReply);
} else {
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(new Uint8Array(payload).buffer);
}
}
+8
View File
@@ -34,6 +34,7 @@ export type WorkerAppMessage =
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
export type WorkerAppReply =
| { type: "buzzer-sequence"; payload: number[] }
| { type: "navigation-sensors"; request: NavigationSensorRequest }
| { type: "open-url"; url: string }
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string }
@@ -183,6 +184,7 @@ export type WorkerAppHostOptions = {
worker: Worker;
navigationSensors?: { handle(request: NavigationSensorRequest): void; stop(): void };
openUrl?: (url: string) => void;
playBuzzerSequence?: (payload: Uint8Array) => Promise<void> | void;
/** Create/refresh a window surface on the compositor (no-op when disconnected). */
configureSurface: (surfaceId: string, visible: boolean, heightMode: WindowHeightMode) => Promise<void>;
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
@@ -240,6 +242,12 @@ export class WorkerAppHost {
const message = event.data as WorkerAppReply | undefined;
if (!message) return;
switch (message.type) {
case "buzzer-sequence":
if (this.openWindows.size) {
void Promise.resolve(this.options.playBuzzerSequence?.(new Uint8Array(message.payload)))
.catch(error => console.warn(`${this.options.appId} buzzer failed: ${error}`));
}
break;
case "navigation-sensors":
if (this.openWindows.size) this.options.navigationSensors?.handle(message.request);
break;