forked from Blendman974/faceclaw
Add support for buzzer sound effects on iOS
This commit is contained in:
1 parent
1412db8e82
commit
e50da605e0
19 files changed
+211
-48
No files matched your search
@@ -17,6 +17,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
|
||||
import { getFont } from "../../graphics/bdffont";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
|
||||
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
|
||||
@@ -33,7 +34,6 @@ import {
|
||||
} from "../../ui/gestures";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
|
||||
const largeFont = getFont("terminus32");
|
||||
const mediumFont = getFont("terminus24");
|
||||
@@ -295,16 +295,13 @@ function inferForeground(window: BlocksWindow, focused: boolean): void {
|
||||
|
||||
/**
|
||||
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
|
||||
* steps on its own timer, and the Java call is safe from the worker thread
|
||||
* (same path as frame submission). Effects never exceed one message, so no
|
||||
* phrase pacing is needed.
|
||||
* steps on its own timer; the platform bridge routes it from the worker.
|
||||
* Effects never exceed one message, so no phrase pacing is needed.
|
||||
*/
|
||||
function playSfx(window: BlocksWindow, steps: Step[]): void {
|
||||
if (!window.soundOn || steps.length === 0) return;
|
||||
try {
|
||||
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
if (!communicator) return;
|
||||
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
|
||||
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
|
||||
} catch (error) {
|
||||
console.warn(`blocks sfx failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -103,7 +103,8 @@ async function startApp(
|
||||
cleanup?: () => void,
|
||||
): Promise<void> {
|
||||
const windowId = `evenhub:app:${nextSerial++}`;
|
||||
const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl);
|
||||
const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl,
|
||||
payload => ctx.actions.playBuzzerSequence(payload));
|
||||
const webView = createEvenHubWebView(session);
|
||||
session.attachWebView({
|
||||
evaluateJs: webView.evaluateJs,
|
||||
|
||||
@@ -65,8 +65,6 @@ import { ensureFineLocationPermission } from "../../native/location-permissions"
|
||||
|
||||
const UPNG = require("upng-js");
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/** OsEventTypeList values (PB ordinals). */
|
||||
const CLICK_EVENT = 0;
|
||||
const SCROLL_TOP_EVENT = 1;
|
||||
@@ -371,6 +369,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
distDir: string,
|
||||
log: (message: string) => void,
|
||||
remoteUrl = "",
|
||||
private readonly sendBuzzerSequence: (payload: Uint8Array) => Promise<void> | void = () => {},
|
||||
) {
|
||||
this.manifest = manifest;
|
||||
this.distDir = distDir;
|
||||
@@ -1164,13 +1163,11 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
private async playBuzzer(rawSteps: unknown[]): Promise<void> {
|
||||
const steps = normalizeBuzzerSteps(rawSteps);
|
||||
if (steps.length === 0) return;
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return;
|
||||
for (let i = 0; i < steps.length; i += CFW_SEQ_MAX) {
|
||||
const chunk = steps.slice(i, i + CFW_SEQ_MAX);
|
||||
const payload = buildSoundSequencePayload(chunk);
|
||||
try {
|
||||
communicator.playBuzzerSequence(payload.buffer);
|
||||
await this.sendBuzzerSequence(payload);
|
||||
} catch (error) {
|
||||
this.log(`evenhub: playBuzzer failed: ${error}`);
|
||||
return;
|
||||
@@ -1274,15 +1271,6 @@ function clampImuFreq(freq: number): number {
|
||||
return Math.min(1000, Math.max(100, snapped));
|
||||
}
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy content into a `preserve` container from the same-named, same-kind
|
||||
* container in the previous page (if any), so a replaceLayout keeps its pixels /
|
||||
|
||||
@@ -31,6 +31,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
|
||||
import { getFont } from "../../graphics/bdffont";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
|
||||
@@ -64,7 +65,6 @@ import {
|
||||
} from "./flappy-sprites";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
|
||||
const largeFont = getFont("terminus32");
|
||||
const mediumFont = getFont("terminus24");
|
||||
@@ -311,7 +311,7 @@ function saveHighScore(window: FlappyWindow): void {
|
||||
|
||||
/**
|
||||
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
|
||||
* steps on its own timer, and the Java call is safe from the worker thread.
|
||||
* steps on its own timer; the platform bridge routes it from the worker.
|
||||
* Minor (frequent) effects are dropped when they'd arrive within the
|
||||
* rate-limit gap; newsworthy ones always play.
|
||||
*/
|
||||
@@ -321,9 +321,7 @@ function playSfx(window: FlappyWindow, steps: Step[], minor = false): void {
|
||||
if (minor && now - window.lastMinorSfxAtMs < MINOR_SFX_MIN_GAP_MS) return;
|
||||
if (minor) window.lastMinorSfxAtMs = now;
|
||||
try {
|
||||
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
if (!communicator) return;
|
||||
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
|
||||
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
|
||||
} catch (error) {
|
||||
console.warn(`flappy sfx failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import { getFont } from "../../graphics/bdffont";
|
||||
import { ensurePreinstalledFonts, installedFontPath } from "../../graphics/installed-fonts";
|
||||
import { TtfFont } from "../../graphics/ttf-font";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
|
||||
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
|
||||
@@ -34,7 +35,6 @@ import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-win
|
||||
import { directionalFallback, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, type InputEvent } from "../../ui/gestures";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
|
||||
ensurePreinstalledFonts();
|
||||
const fontPath = installedFontPath("Roboto-Regular.ttf");
|
||||
@@ -206,14 +206,12 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
|
||||
/**
|
||||
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
|
||||
* steps on its own timer, and the Java call is safe from the worker thread.
|
||||
* steps on its own timer; the platform bridge routes it from the worker.
|
||||
*/
|
||||
function playSfx(window: FreecellWindow, steps: Step[]): void {
|
||||
if (!window.soundOn || steps.length === 0) return;
|
||||
try {
|
||||
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
if (!communicator) return;
|
||||
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
|
||||
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
|
||||
} catch (error) {
|
||||
console.warn(`freecell sfx failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import { getFont } from "../../graphics/bdffont";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
|
||||
import { loadSoundEnabled, saveSoundEnabled } from "../../ui/sound-setting";
|
||||
@@ -43,7 +44,6 @@ import {
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
|
||||
const largeFont = getFont("terminus32");
|
||||
const mediumFont = getFont("terminus24");
|
||||
@@ -278,14 +278,12 @@ function saveDifficultyIndex(window: MinesweeperWindow): void {
|
||||
|
||||
/**
|
||||
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
|
||||
* steps on its own timer, and the Java call is safe from the worker thread.
|
||||
* steps on its own timer; the platform bridge routes it from the worker.
|
||||
*/
|
||||
function playSfx(window: MinesweeperWindow, steps: Step[]): void {
|
||||
if (!window.soundOn || steps.length === 0) return;
|
||||
try {
|
||||
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
if (!communicator) return;
|
||||
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
|
||||
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
|
||||
} catch (error) {
|
||||
console.warn(`minesweeper sfx failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire";
|
||||
import { getFont } from "../../graphics/bdffont";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { playWorkerBuzzerSequence } from "../../native/worker-buzzer";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects";
|
||||
@@ -44,7 +45,6 @@ import {
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
|
||||
const largeFont = getFont("terminus32");
|
||||
const mediumFont = getFont("terminus24");
|
||||
@@ -429,7 +429,7 @@ function saveHighScore(window: PinballWindow): void {
|
||||
|
||||
/**
|
||||
* Fire a buzzer effect. Non-blocking: the firmware's sequencer plays the
|
||||
* steps on its own timer, and the Java call is safe from the worker thread.
|
||||
* steps on its own timer; the platform bridge routes it from the worker.
|
||||
* Minor (frequent) effects are dropped when they'd arrive within the
|
||||
* rate-limit gap; newsworthy ones always play.
|
||||
*/
|
||||
@@ -439,9 +439,7 @@ function playSfx(window: PinballWindow, steps: Step[], minor = false): void {
|
||||
if (minor && now - window.lastMinorSfxAtMs < MINOR_SFX_MIN_GAP_MS) return;
|
||||
if (minor) window.lastMinorSfxAtMs = now;
|
||||
try {
|
||||
const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
if (!communicator) return;
|
||||
communicator.playBuzzerSequence(buildSoundSequencePayload(steps).buffer);
|
||||
playWorkerBuzzerSequence(buildSoundSequencePayload(steps));
|
||||
} catch (error) {
|
||||
console.warn(`pinball sfx failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -427,6 +427,13 @@ export class GlassesSession {
|
||||
if (this.state.phase !== 'connected') return
|
||||
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
|
||||
}
|
||||
/** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */
|
||||
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
if (this.state.phase !== 'connected' || !this.layoutCreated || payload.length < 3) return
|
||||
const generation = this.generation
|
||||
try { await this.requestCfw('left', new Uint8Array(payload)) }
|
||||
catch (error) { if (generation === this.generation) this.fail(error, true) }
|
||||
}
|
||||
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
|
||||
const token = ++this.microphoneToken, generation = this.generation
|
||||
|
||||
@@ -43,6 +43,8 @@ import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell
|
||||
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting } from '../ui/dashboard-settings'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state'
|
||||
import { findSoundEffect, playSoundEffect } from '../ui/sound-effects'
|
||||
|
||||
/** iOS host for the shared app registry, shell, compositor and BLE session. */
|
||||
export class IosPreviewController {
|
||||
@@ -96,10 +98,12 @@ export class IosPreviewController {
|
||||
private glassesLocked = false
|
||||
private lockEnabled = lockScreenEnabledSetting.get()
|
||||
private session: GlassesSession | null = null
|
||||
private acknowledgedFrames = 0
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
playBuzzerSequence: payload => this.session?.playBuzzerSequence(payload),
|
||||
requestRender: () => this.requestShellRender(),
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
@@ -457,6 +461,7 @@ export class IosPreviewController {
|
||||
? new IosNavigationSensors(event => worker.postMessage({ type: 'navigation-sensors', event })) : undefined
|
||||
const host = new WorkerAppHost({
|
||||
appId, worker, navigationSensors,
|
||||
playBuzzerSequence: payload => this.actions.playBuzzerSequence(payload),
|
||||
openUrl: url => { void Utils.openUrl(url) },
|
||||
configureSurface: async (id, visible, heightMode) => {
|
||||
this.compositor.configureSurface(id, { ...appViewportRect(heightMode, appId), zOrder: 0, transparency: 'opaque' })
|
||||
@@ -492,6 +497,7 @@ export class IosPreviewController {
|
||||
if (error) { this.onError(error); return }
|
||||
if (!this.session) {
|
||||
this.session = new GlassesSession(iosBluetooth(), state => {
|
||||
this.maybePlayWelcomeSound(state)
|
||||
if (state.phase !== 'connected' && state.phase !== 'connecting') this.glassesWorn = null
|
||||
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
|
||||
@@ -517,6 +523,17 @@ export class IosPreviewController {
|
||||
await this.session.start(addresses)
|
||||
}
|
||||
async disconnect(): Promise<void> { await this.session?.stop() }
|
||||
private maybePlayWelcomeSound(state: SessionState): void {
|
||||
const firstNewFrame = state.frames > this.acknowledgedFrames
|
||||
this.acknowledgedFrames = state.frames
|
||||
// Match Android: wait for an acknowledged frame before consuming the jingle.
|
||||
if (!firstNewFrame || state.phase !== 'connected' || !isWelcomeSoundPending()) return
|
||||
setWelcomeSoundPending(false)
|
||||
const effect = findSoundEffect('questcomplete')
|
||||
if (effect) void playSoundEffect(effect, payload => this.actions.playBuzzerSequence(payload),
|
||||
ms => new Promise(resolve => setTimeout(resolve, ms)))
|
||||
.catch(error => this.logBluetooth(`Welcome sound failed: ${error}`))
|
||||
}
|
||||
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (this.glassesLocked) return false
|
||||
|
||||
@@ -7,9 +7,8 @@ declare const com: any;
|
||||
* through Java statics because those are shared across isolates, unlike any
|
||||
* JS-side state.
|
||||
*
|
||||
* Frame submission only. Glasses-hardware calls (e.g. playBuzzerSequence)
|
||||
* should keep using FaceclawBleCommunicator.getActive() directly, so they
|
||||
* skip cleanly when nothing is connected.
|
||||
* Frame submission only. Buzzer effects use worker-buzzer, which routes to
|
||||
* the platform BLE session and skips playback when disconnected.
|
||||
*/
|
||||
export function getActiveDisplay(): any {
|
||||
return (
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { WorkerAppReply } from "../ui/shell/worker-window";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/** Workers use the main-thread BLE session on iOS and the native queue on Android. */
|
||||
export function playWorkerBuzzerSequence(payload: Uint8Array): void {
|
||||
if (global.isIOS) {
|
||||
global.postMessage({ type: "buzzer-sequence", payload: Array.from(payload) } satisfies WorkerAppReply);
|
||||
} else {
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(new Uint8Array(payload).buffer);
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ export type WorkerAppMessage =
|
||||
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
|
||||
|
||||
export type WorkerAppReply =
|
||||
| { type: "buzzer-sequence"; payload: number[] }
|
||||
| { type: "navigation-sensors"; request: NavigationSensorRequest }
|
||||
| { type: "open-url"; url: string }
|
||||
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string }
|
||||
@@ -183,6 +184,7 @@ export type WorkerAppHostOptions = {
|
||||
worker: Worker;
|
||||
navigationSensors?: { handle(request: NavigationSensorRequest): void; stop(): void };
|
||||
openUrl?: (url: string) => void;
|
||||
playBuzzerSequence?: (payload: Uint8Array) => Promise<void> | void;
|
||||
/** Create/refresh a window surface on the compositor (no-op when disconnected). */
|
||||
configureSurface: (surfaceId: string, visible: boolean, heightMode: WindowHeightMode) => Promise<void>;
|
||||
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
|
||||
@@ -240,6 +242,12 @@ export class WorkerAppHost {
|
||||
const message = event.data as WorkerAppReply | undefined;
|
||||
if (!message) return;
|
||||
switch (message.type) {
|
||||
case "buzzer-sequence":
|
||||
if (this.openWindows.size) {
|
||||
void Promise.resolve(this.options.playBuzzerSequence?.(new Uint8Array(message.payload)))
|
||||
.catch(error => console.warn(`${this.options.appId} buzzer failed: ${error}`));
|
||||
}
|
||||
break;
|
||||
case "navigation-sensors":
|
||||
if (this.openWindows.size) this.options.navigationSensors?.handle(message.request);
|
||||
break;
|
||||
|
||||
Reference in new issue
Block a user