diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 0b399ca7..5460bfc4 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -361,23 +361,35 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { interruptibleSleep.interrupt(); } - /** Play one quantized buzzer tone via CFW load_image_z mode 5 kind 1. */ - public void playBuzzerNote(int note, int oct, int beat) { + /** + * Play a tone sequence via CFW load_image_z mode 5 kind 4. The payload is + * the complete wire buffer ([5][4][nSteps][freqLo,freqHi,duty,msLo,msHi]*n, + * up to 48 steps), built on the TS side; it rides the arbitrary-payload + * image path like the other mode-5 controls. + */ + public void playBuzzerSequence(java.nio.ByteBuffer payload) { synchronized (lock) { if (!running || !sessionReady || !fixedLayoutCreated) { - logLine("skip buzzer note; session not ready"); + logLine("skip buzzer sequence; session not ready"); + return; + } + byte[] bytes = new byte[payload == null ? 0 : payload.remaining()]; + if (payload != null) { + payload.get(bytes); + } + if (bytes.length < 3) { + logLine("skip buzzer sequence; empty payload"); return; } - byte[] payload = BleProtocol.buildBuzzerNotePayload(note, oct, beat); OutboundMessage message = messageBuilder.imagePayload( DASHBOARD_TILE, nextMapSessionId(), - payload, - "buzzer note=" + note + " oct=" + oct + " beat=" + beat, + bytes, + "buzzer sequence " + bytes.length + "B", connectionOptions.sendImagesToLeft ); message.onTimeout = () -> { - handleTransportFailure("buzzer note ack timeout"); + handleTransportFailure("buzzer sequence ack timeout"); }; pendingMessages.addLast(message); logLine("queue " + message.label); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java index e7ef0d07..0f45003f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java @@ -3,13 +3,20 @@ package com.faceclaw.app; import android.content.ComponentName; import android.content.Context; import android.content.Intent; +import android.graphics.Bitmap; +import android.media.MediaDescription; import android.media.MediaMetadata; import android.media.session.MediaController; +import android.media.session.MediaSession; import android.media.session.MediaSessionManager; import android.media.session.PlaybackState; import android.os.Handler; import android.os.Looper; import android.provider.Settings; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONObject; import java.util.List; @@ -35,6 +42,13 @@ public class FaceclawMediaController { } } + @Override + public void onQueueChanged(List queue) { + synchronized (lock) { + emitStateLocked(); + } + } + @Override public void onSessionDestroyed() { synchronized (lock) { @@ -149,6 +163,107 @@ public class FaceclawMediaController { } } + public void skipToQueueItem(long queueId) { + synchronized (lock) { + if (activeController != null) { + activeController.getTransportControls().skipToQueueItem(queueId); + } + } + } + + /** + * Album art for the active session's current item, grayscale, scaled to + * fit within maxSize x maxSize preserving aspect. Returns + * [widthLo, widthHi, heightLo, heightHi, pixels...] or an empty array + * when no art is available. + */ + public byte[] getAlbumArtGray(int maxSize) { + Bitmap art; + synchronized (lock) { + if (activeController == null) { + return new byte[0]; + } + MediaMetadata metadata = activeController.getMetadata(); + if (metadata == null) { + return new byte[0]; + } + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ALBUM_ART); + if (art == null) { + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ART); + } + if (art == null) { + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_DISPLAY_ICON); + } + } + if (art == null || art.getWidth() <= 0 || art.getHeight() <= 0 || maxSize <= 0) { + return new byte[0]; + } + try { + float scale = Math.min(1f, Math.min( + (float) maxSize / art.getWidth(), + (float) maxSize / art.getHeight())); + int width = Math.max(1, Math.round(art.getWidth() * scale)); + int height = Math.max(1, Math.round(art.getHeight() * scale)); + Bitmap scaled = Bitmap.createScaledBitmap(art, width, height, true); + if (scaled.getConfig() == Bitmap.Config.HARDWARE) { + scaled = scaled.copy(Bitmap.Config.ARGB_8888, false); + } + int[] pixels = new int[width * height]; + scaled.getPixels(pixels, 0, width, 0, 0, width, height); + byte[] out = new byte[4 + width * height]; + out[0] = (byte) (width & 0xff); + out[1] = (byte) ((width >> 8) & 0xff); + out[2] = (byte) (height & 0xff); + out[3] = (byte) ((height >> 8) & 0xff); + for (int i = 0; i < pixels.length; i++) { + int p = pixels[i]; + int r = (p >> 16) & 0xff; + int g = (p >> 8) & 0xff; + int b = p & 0xff; + out[4 + i] = (byte) ((r * 299 + g * 587 + b * 114) / 1000); + } + return out; + } catch (Exception e) { + Log.w("FaceclawMedia", "album art conversion failed", e); + return new byte[0]; + } + } + + /** + * The active session's queue (playlist) as JSON: + * [{"id": long, "title": string, "active": bool}, ...]. Empty string when + * the player exposes no queue. + */ + public String getQueueJson() { + synchronized (lock) { + if (activeController == null) { + return ""; + } + List queue = activeController.getQueue(); + if (queue == null || queue.isEmpty()) { + return ""; + } + PlaybackState state = activeController.getPlaybackState(); + long activeId = state == null ? -1 : state.getActiveQueueItemId(); + try { + JSONArray out = new JSONArray(); + for (MediaSession.QueueItem item : queue) { + MediaDescription description = item.getDescription(); + CharSequence title = description == null ? null : description.getTitle(); + JSONObject entry = new JSONObject(); + entry.put("id", item.getQueueId()); + entry.put("title", title == null ? "" : title.toString()); + entry.put("active", item.getQueueId() == activeId); + out.put(entry); + } + return out.toString(); + } catch (Exception e) { + Log.w("FaceclawMedia", "queue serialization failed", e); + return ""; + } + } + } + public void openNotificationAccessSettings() { Intent intent = new Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS); intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java index 3b1b7ce7..42680670 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java @@ -121,14 +121,6 @@ public class BleProtocol { return wrapEvenHub(5, magic, 9, concat(inner)); } - /** CFW load_image_z mode 5 kind 1: play one quantized buzzer note. */ - public static byte[] buildBuzzerNotePayload(int note, int oct, int beat) { - int clampedNote = Math.max(1, Math.min(7, note)); - int clampedOct = Math.max(0, Math.min(3, oct)); - int clampedBeat = Math.max(1, Math.min(255, beat)); - return new byte[] {5, 1, (byte) clampedNote, (byte) clampedOct, (byte) clampedBeat}; - } - public static byte[] buildImageRawData(ImageTileOptions tile, int sessionId, int totalSize, ImageFragment fragment, int magic) { List inner = new ArrayList<>(); inner.add(encodeVarintField(1, tile.containerId)); diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index fa9c6a03..4a777beb 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -54,6 +54,7 @@ import { NIGHTSCOUT_SURFACE_ID, NIGHTSCOUT_WINDOW_ID, } from "../ui/shell/nightscout-app"; +import { createMusicAppWindow, MUSIC_SURFACE_ID, MUSIC_WINDOW_ID } from "../ui/shell/music-app"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { APP_VIEWPORT } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; @@ -234,7 +235,7 @@ class DashboardController { setTranscribeRenderActive: (active: boolean) => this.setTranscribeRenderActive(active), startVoiceCapture: () => this.startVoiceCapture(), stopVoiceCapture: () => this.stopVoiceCapture(), - playBuzzerNote: (note: number, oct: number, beat: number) => this.playBuzzerNote(note, oct, beat), + playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; setDashboardActions({ @@ -279,6 +280,7 @@ class DashboardController { { appId: "stopwatch", label: "Stopwatch" }, { appId: "terminal", label: "Terminal" }, { appId: "teleprompt", label: "Teleprompt" }, + { appId: "music", label: "Music" }, { appId: "nightscout", label: "Nightscout" }, { appId: "notifications", label: "Notifications" }, { appId: "debug-tests", label: "Debug tests" }, @@ -1138,6 +1140,10 @@ class DashboardController { await this.launchInProcessApp(NIGHTSCOUT_WINDOW_ID, NIGHTSCOUT_SURFACE_ID, createNightscoutAppWindow); return; } + if (appId === "music") { + await this.launchInProcessApp(MUSIC_WINDOW_ID, MUSIC_SURFACE_ID, createMusicAppWindow); + return; + } if (appId === "teleprompt") { await this.launchInProcessApp(TELEPROMPT_WINDOW_ID, TELEPROMPT_SURFACE_ID, (options) => createTelepromptBrowserWindow({ @@ -1320,11 +1326,11 @@ class DashboardController { } } - private async playBuzzerNote(note: number, oct: number, beat: number): Promise { + private async playBuzzerSequence(payload: Uint8Array): Promise { if (this.phase !== "connected" || !this.communicator) { return; } - await this.communicator.playBuzzerNote(note, oct, beat); + await this.communicator.playBuzzerSequence(payload); } private setTranscribeRenderActive(active: boolean): void { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 6244f141..7962d8c8 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -421,13 +421,11 @@ export class FaceclawCommunicatorBridge { return this.enqueueJavaCall(() => Boolean(this.communicator.sendShutdown(exitMode))); } - async playBuzzerNote(note: number, oct: number, beat: number): Promise { + /** Play a CFW mode-5 kind-4 tone sequence (complete wire payload). */ + async playBuzzerSequence(payload: Uint8Array): Promise { + const snapshot = new Uint8Array(payload); await this.enqueueJavaCall(() => { - this.communicator.playBuzzerNote( - Math.round(note), - Math.round(oct), - Math.round(beat), - ); + this.communicator.playBuzzerSequence(snapshot.buffer); }); } diff --git a/app/native/media-controller.ts b/app/native/media-controller.ts index f5562b51..428fe15c 100644 --- a/app/native/media-controller.ts +++ b/app/native/media-controller.ts @@ -1,7 +1,15 @@ import { Utils } from "@nativescript/core"; +import { GrayImage } from "../graphics/image"; +import { toUint8Array } from "../util/array-util"; declare const com: any; +export type MediaQueueItem = { + id: number; + title: string; + active: boolean; +}; + export type MediaPlaybackState = | "notification-access-required" | "idle" @@ -89,6 +97,50 @@ export class FaceclawMediaControllerBridge { this.controller?.openNotificationAccessSettings(); } + /** Jump playback to a queue item (see getQueue). */ + async skipToQueueItem(id: number): Promise { + this.ensureController(); + this.controller?.skipToQueueItem(id); + } + + /** + * Album art for the current item, grayscale, scaled to fit maxSize; null + * when the player provides none. + */ + getAlbumArt(maxSize: number): GrayImage | null { + if (!global.isAndroid) return null; + this.ensureController(); + if (!this.controller) return null; + const bytes = toUint8Array(this.controller.getAlbumArtGray(Math.round(maxSize))); + if (bytes.length < 4) return null; + const width = bytes[0]! | (bytes[1]! << 8); + const height = bytes[2]! | (bytes[3]! << 8); + if (width <= 0 || height <= 0 || bytes.length < 4 + width * height) return null; + const image = new GrayImage(width, height, 0); + image.pixels.set(bytes.subarray(4, 4 + width * height)); + return image; + } + + /** The player's queue (playlist), empty when the player exposes none. */ + getQueue(): MediaQueueItem[] { + if (!global.isAndroid) return []; + this.ensureController(); + if (!this.controller) return []; + const json = String(this.controller.getQueueJson() ?? ""); + if (!json) return []; + try { + const raw = JSON.parse(json) as Array<{ id?: number; title?: string; active?: boolean }>; + return raw.map((item) => ({ + id: Number(item.id ?? -1), + title: String(item.title ?? ""), + active: Boolean(item.active), + })); + } catch (error) { + console.warn(`media queue parse failed: ${error}`); + return []; + } + } + private ensureController(): void { if (!global.isAndroid || this.controller) return; const context = Utils.android.getApplicationContext(); diff --git a/app/ui/apps/buzzer-demo.ts b/app/ui/apps/buzzer-demo.ts index 78b60259..159651d1 100644 --- a/app/ui/apps/buzzer-demo.ts +++ b/app/ui/apps/buzzer-demo.ts @@ -1,57 +1,89 @@ import { getDefaultSmallFont } from "../../graphics/bdffont"; import { GrayImage } from "../../graphics/image"; -import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; +import { clamp } from "../../util/numeric-util"; +import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; +import { + buildSoundSequencePayload, + CFW_SEQ_MAX, + effectPhrases, + phraseDurationMs, + SOUND_EFFECTS, + type SoundEffect, +} from "./sound-effects"; -const NOTE_NAMES = ["C", "D", "E", "F", "G", "A", "B"] as const; -const DEFAULT_BEAT = 4; // ~250ms at 62ms units +const HEADER_HEIGHT = 30; +const ROW_HEIGHT = 16; +const LIST_X = 20; +const FOOTER_HEIGHT = 22; +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Buzzer demo: pick a sound effect from the catalog and play it on the + * glasses' piezo. Multi-phrase effects are paced phrase-by-phrase so each + * sequencer message lands as the previous phrase finishes. + */ export class BuzzerDemoLayer implements Layer { - private note = 1; - private oct = 2; + private selectedIndex = 0; + private scrollRow = 0; + private playing: string | null = null; - paint(ctx: LayerContext, _paintBelow: () => GrayImage): GrayImage { - // Sized to the hosting stack (the Debug tests app viewport). + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); const { width, height } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); - const font = getDefaultSmallFont(); - image.drawText(font, 24, 20, "Buzzer demo", 200); - image.drawText(font, 24, 44, `Note: ${NOTE_NAMES[this.note - 1]} Octave: ${this.oct}`, 180); - image.drawText(font, 24, 68, "Scroll: change note", 150); - image.drawText(font, 24, 84, "Click: play note", 150); - image.drawText(font, 24, height - 24, "Double-click: back", 130); + image.drawText(font, 20, 8, "Buzzer demo", 220); + const status = this.playing ? `Playing: ${this.playing}` : "Click to play"; + image.drawText(font, width - 24 - font.measureText(status), 8, status, 140); - const keyWidth = 64; - const keyY = 130; - const keyX = Math.floor((width - keyWidth * NOTE_NAMES.length) / 2); - for (let i = 0; i < NOTE_NAMES.length; i++) { - const selected = i + 1 === this.note; - const x = keyX + i * keyWidth; - image.drawRect(x + 2, keyY, keyWidth - 4, 48, selected ? 220 : 80); - const label = NOTE_NAMES[i]!; - image.drawText( - font, - x + Math.max(4, Math.floor((keyWidth - font.measureText(label)) / 2)), - keyY + 18, - label, - selected ? 0 : 200, - ); + const listHeight = height - HEADER_HEIGHT - FOOTER_HEIGHT; + const visibleRows = Math.max(1, (listHeight / ROW_HEIGHT) | 0); + if (this.selectedIndex < this.scrollRow) { + this.scrollRow = this.selectedIndex; + } else if (this.selectedIndex >= this.scrollRow + visibleRows) { + this.scrollRow = this.selectedIndex - visibleRows + 1; + } + this.scrollRow = clamp(this.scrollRow, 0, Math.max(0, SOUND_EFFECTS.length - visibleRows)); + + const lastVisible = Math.min(SOUND_EFFECTS.length, this.scrollRow + visibleRows); + for (let index = this.scrollRow; index < lastVisible; index++) { + const effect = SOUND_EFFECTS[index]!; + const y = HEADER_HEIGHT + (index - this.scrollRow) * ROW_HEIGHT; + const selected = index === this.selectedIndex; + if (selected) { + image.fillRoundedRect(LIST_X - 6, y - 1, width - 2 * LIST_X + 12, ROW_HEIGHT - 1, 15, 4); + image.drawRoundedRect(LIST_X - 6, y - 1, width - 2 * LIST_X + 12, ROW_HEIGHT - 1, 45, 4); + } + image.drawText(font, LIST_X, y + 1, effect.name, selected ? 255 : 200); + image.drawText(font, LIST_X + 110, y + 1, effect.desc, selected ? 180 : 120); } + image.drawText(font, 20, height - 16, "Scroll: select Click: play Double-click: back", 110); return image; } async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { switch (event.type) { case "scroll-up": - this.advanceNote(1); + this.selectedIndex = Math.max(0, this.selectedIndex - 1); return; case "scroll-down": - this.advanceNote(-1); + this.selectedIndex = Math.min(SOUND_EFFECTS.length - 1, this.selectedIndex + 1); return; - case "click": - await ctx.actions.playBuzzerNote(this.note, this.oct, DEFAULT_BEAT); + case "click": { + const effect = SOUND_EFFECTS[clamp(this.selectedIndex, 0, SOUND_EFFECTS.length - 1)]; + if (effect && !this.playing) { + // Fire-and-forget: playback paces itself with sleeps between + // phrases; awaiting it here would stall input for seconds. + void this.playEffect(effect, ctx.actions).catch((error) => { + console.warn(`buzzer effect failed: ${error}`); + }); + } return; + } case "double-click": ctx.stack.pop(); return; @@ -60,18 +92,21 @@ export class BuzzerDemoLayer implements Layer { } } - private advanceNote(delta: number): void { - let next = this.note + delta; - if (next > 7) { - this.note = 1; - this.oct = Math.min(3, this.oct + 1); - return; + private async playEffect(effect: SoundEffect, actions: LayerActions): Promise { + this.playing = effect.name; + actions.requestRender(); + try { + for (const phrase of effectPhrases(effect.make())) { + // Safety net: chop any over-long phrase into <=48-step messages. + for (let index = 0; index < phrase.length; index += CFW_SEQ_MAX) { + const chunk = phrase.slice(index, index + CFW_SEQ_MAX); + await actions.playBuzzerSequence(buildSoundSequencePayload(chunk)); + await sleep(phraseDurationMs(chunk)); + } + } + } finally { + this.playing = null; + actions.requestRender(); } - if (next < 1) { - this.note = 7; - this.oct = Math.max(0, this.oct - 1); - return; - } - this.note = next; } } diff --git a/app/ui/apps/music-controller.ts b/app/ui/apps/music-controller.ts index 404e2f5a..3601f090 100644 --- a/app/ui/apps/music-controller.ts +++ b/app/ui/apps/music-controller.ts @@ -3,7 +3,6 @@ import { getDefaultSmallFont } from "../../graphics/bdffont"; import { GrayImage } from "../../graphics/image"; import { wrapText } from "../../graphics/textwrap"; import { mediaControllerBridge, type MediaControllerState } from "../../native/media-controller"; -import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; export class MusicControllerDashboardPlugin extends DashboardPlugin { @@ -38,81 +37,8 @@ export class MusicControllerDashboardPlugin extends DashboardPlugin { image.drawText(font, bounds.x + 10, bounds.y + bounds.height - 18, playbackLabel(media), 130); } - override createFullscreenLayer(getState: () => DashboardPluginState): Layer { - return new MusicControllerLayer(getState); - } } -class MusicControllerLayer implements Layer { - constructor(private readonly getState: () => DashboardPluginState) {} - - paint(ctx: LayerContext): GrayImage { - const font = getDefaultSmallFont(); - const image = new GrayImage(576, 288, 0); - const media = this.getState().media; - image.drawRect(12, 12, 552, 264, 52); - image.drawText(font, 22, 16, "Music controller", 220); - - if (!media.accessEnabled) { - const lines = wrapText( - font, - "Notification access is required before Android will expose active media sessions to Faceclaw. Click to open settings.", - 520, - ); - for (let i = 0; i < lines.length; i++) { - image.drawText(font, 22, 44 + i * 14, lines[i]!, 180); - } - image.drawText(font, 22, 252, "Click: open settings Double-click: back", 110); - return image; - } - - if (!media.available) { - image.drawText(font, 22, 44, "No active media session.", 180); - image.drawText(font, 22, 58, "Start playback in another app,", 180); - image.drawText(font, 22, 72, "then reopen this card.", 180); - image.drawText(font, 22, 252, "Double-click: back", 110); - return image; - } - - image.drawText(font, 22, 44, media.title || "Unknown title", 220); - image.drawText(font, 22, 60, media.artist || media.album || "Unknown artist", 180); - image.drawText(font, 22, 76, media.packageName, 130); - image.drawText(font, 22, 104, `State: ${playbackLabel(media)}`, 180); - image.drawText(font, 22, 132, "Click: play/pause", 180); - image.drawText(font, 22, 146, "Scroll up: previous", 180); - image.drawText(font, 22, 160, "Scroll down: next", 180); - image.drawText(font, 22, 252, "Double-click: back", 110); - return image; - } - - async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { - const media = this.getState().media; - switch (event.type) { - case "click": - if (!media.accessEnabled) { - mediaControllerBridge.openNotificationAccessSettings(); - } else if (media.canPlayPause) { - await mediaControllerBridge.playPause(); - } - return; - case "scroll-up": - if (media.canSkipPrevious) { - await mediaControllerBridge.skipPrevious(); - } - return; - case "scroll-down": - if (media.canSkipNext) { - await mediaControllerBridge.skipNext(); - } - return; - case "double-click": - ctx.stack.pop(); - return; - default: - return; - } - } -} function playbackLabel(media: MediaControllerState): string { switch (media.playbackState) { diff --git a/app/ui/apps/sound-effects.ts b/app/ui/apps/sound-effects.ts new file mode 100644 index 00000000..aaa6b13d --- /dev/null +++ b/app/ui/apps/sound-effects.ts @@ -0,0 +1,192 @@ +/** + * Sound-effect catalog for the CFW buzzer tone sequencer (load_image_z mode 5 + * kind 4). Ported from g2-kit-unofficial/examples/tunes.ts. + * + * The G2 "speaker" is a single monophonic PWM piezo buzzer. One message + * carries up to 48 back-to-back {freq, duty, ms} steps that the firmware + * plays on its own one-shot timer (non-blocking, auto-stops after the last + * step). Effects longer than 48 steps split into phrases, each its own + * message sent back-to-back, broken at musical rests so the tiny + * inter-message gap is inaudible. + * + * Piezo reality check: the element resonates best around ~1-4 kHz; below + * ~150 Hz it is faint, above ~8 kHz it thins out. duty ~50% is the fullest + * square wave; lower duty reads as thinner and quieter. + */ + +/** One PWM segment; duty 0 means "rest" (silence) for `ms`. */ +export type Step = { freq: number; duty?: number; ms: number }; +/** One or more phrases; each phrase is one sequencer message. */ +export type Effect = Step[] | Step[][]; + +/** Firmware cap: steps per mode-5 kind-4 message. */ +export const CFW_SEQ_MAX = 48; + +/** Build the wire buffer: [5][4][nSteps][ freqLo,freqHi,duty,msLo,msHi ]*n. */ +export function buildSoundSequencePayload(steps: Step[]): Uint8Array { + const count = Math.min(steps.length, CFW_SEQ_MAX); + const out = new Uint8Array(3 + count * 5); + out[0] = 0x05; + out[1] = 0x04; + out[2] = count; + for (let index = 0; index < count; index++) { + const { freq, duty = 50, ms } = steps[index]!; + const offset = 3 + index * 5; + const clampedFreq = Math.max(1, Math.min(20000, Math.round(freq))); + const clampedDuty = Math.max(0, Math.min(100, Math.round(duty))); + const clampedMs = Math.max(1, Math.min(65535, Math.round(ms))); + out[offset] = clampedFreq & 0xff; + out[offset + 1] = (clampedFreq >> 8) & 0xff; + out[offset + 2] = clampedDuty; + out[offset + 3] = clampedMs & 0xff; + out[offset + 4] = (clampedMs >> 8) & 0xff; + } + return out; +} + +export function phraseDurationMs(steps: Step[]): number { + return steps.reduce((total, step) => total + step.ms, 0); +} + +/** Normalize an Effect into its list of phrases. */ +export function effectPhrases(effect: Effect): Step[][] { + return Array.isArray(effect[0]) ? (effect as Step[][]) : [effect as Step[]]; +} + +// ============================ musical helpers =============================== +// Equal-tempered pitch from a name like "A4", "C#5", "Bb3". A4 = 440 Hz. +const SEMI: Record = { C: 0, D: 2, E: 4, F: 5, G: 7, A: 9, B: 11 }; +function hz(name: string): number { + const match = /^([A-Ga-g])([#b]?)(-?\d)$/.exec(name.trim()); + if (!match) throw new Error(`bad note: ${name}`); + let semitone = SEMI[match[1]!.toUpperCase()]!; + if (match[2] === "#") semitone += 1; + else if (match[2] === "b") semitone -= 1; + const midi = semitone + (parseInt(match[3]!, 10) + 1) * 12; + return Math.round(440 * 2 ** ((midi - 69) / 12)); +} + +// Compact melody DSL: whitespace-separated "NOTE/ms" tokens. "." or "r" = rest. +function notes(dsl: string, defMs = 140, duty = 50): Step[] { + return dsl.trim().split(/\s+/).map((token) => { + const [note, durationText] = token.split("/"); + const ms = durationText ? parseInt(durationText, 10) : defMs; + if (note === "." || note === "r" || note === "rest") return { freq: 1, duty: 0, ms }; + return { freq: hz(note!), duty, ms }; + }); +} + +// Geometric (log-linear = constant perceived glide) frequency sweep across +// `ms`, approximated by `steps` short constant-tone segments. +function sweep(from: number, to: number, ms: number, steps = 16, duty = 50): Step[] { + steps = Math.max(2, Math.min(CFW_SEQ_MAX, steps)); + const per = Math.max(1, ms / steps); + const out: Step[] = []; + for (let index = 0; index < steps; index++) { + const freq = from * (to / from) ** (index / (steps - 1)); + out.push({ freq, duty, ms: per }); + } + return out; +} + +const cat = (...parts: Step[][]): Step[] => parts.flat(); +const rest = (ms: number): Step => ({ freq: 1, duty: 0, ms }); +const rep = (steps: Step[], n: number): Step[] => Array.from({ length: n }, () => steps).flat(); + +// ================================ catalog =================================== +// Each effect keeps every phrase <= 48 steps. Ordered roughly best-first. +export type SoundEffect = { name: string; desc: string; make: () => Effect }; + +export const SOUND_EFFECTS: SoundEffect[] = [ + // --- game SFX --- + { name: "coin", desc: "Mario coin: blip then a bright ring", make: () => + notes("B5/70 E6/520") }, + { name: "powerup", desc: "ascending arpeggio glissando", make: () => { + const out: Step[] = []; + const cells = ["G4", "C5", "E5", "G5", "C6", "E6", "G6"]; + for (const cell of cells) out.push({ freq: hz(cell), duty: 50, ms: 42 }); + out.push(rest(20), { freq: hz("G6"), duty: 50, ms: 60 }, + { freq: hz("C7"), duty: 50, ms: 120 }); + return out; + } }, + { name: "oneup", desc: "Mario 1-UP jingle", make: () => + notes("E5/130 G5/130 E6/130 C6/130 D6/130 G6/300") }, + { name: "laser", desc: "downward pew, 1.8k -> 320 Hz", make: () => + sweep(1800, 320, 190, 26, 50) }, + { name: "pew", desc: "short high blaster", make: () => + sweep(2800, 700, 80, 12, 45) }, + { name: "zap", desc: "up-then-down electric snap", make: () => + cat(sweep(400, 2300, 55, 9), sweep(2300, 260, 95, 13)) }, + { name: "warp", desc: "teleport: rising whoosh with vibrato tail", make: () => + cat(sweep(220, 2600, 240, 30, 45), + [{ freq: 2600, duty: 50, ms: 30 }, { freq: 2200, duty: 50, ms: 30 }, + { freq: 2600, duty: 50, ms: 30 }, { freq: 2000, duty: 50, ms: 40 }]) }, + { name: "explosion", desc: "descending rumble (piezo-faint low end)", make: () => + cat(sweep(900, 150, 380, 30, 60), [rest(30), { freq: 160, duty: 70, ms: 120 }]) }, + + // --- UI / notification --- + { name: "success", desc: "rising major arpeggio -> resolve", make: () => + notes("C5/90 E5/90 G5/90 C6/240") }, + { name: "notify", desc: "two-tone up chime", make: () => + notes("A5/90 E6/260") }, + { name: "chime", desc: "two-tone down chime (softer)", make: () => + notes("E6/90 A5/300", 140, 45) }, + { name: "doorbell", desc: "ding-dong", make: () => + notes("E5/300 ./40 C5/560") }, + { name: "error", desc: "harsh low descending two-tone", make: () => + [{ freq: 233, duty: 55, ms: 200 }, rest(20), { freq: 175, duty: 55, ms: 300 }] }, + { name: "deny", desc: "double low buzz (nope)", make: () => + [{ freq: 150, duty: 60, ms: 110 }, rest(60), { freq: 150, duty: 60, ms: 110 }] }, + { name: "tick", desc: "tiny UI click", make: () => + [{ freq: 2000, duty: 22, ms: 6 }] }, + { name: "doublebeep", desc: "two clean beeps", make: () => + [{ freq: 1568, duty: 50, ms: 60 }, rest(60), { freq: 1568, duty: 50, ms: 60 }] }, + { name: "sonar", desc: "single sonar ping", make: () => + [{ freq: 1400, duty: 50, ms: 40 }, { freq: 1350, duty: 30, ms: 120 }] }, + + // --- alarms / attention --- + { name: "alarm", desc: "beep-beep-beep-beep-beep", make: () => + rep([{ freq: 1000, duty: 55, ms: 120 }, rest(90)], 5) }, + { name: "siren", desc: "police wail (two-tone x5)", make: () => + rep([{ freq: 900, duty: 50, ms: 190 }, { freq: 680, duty: 50, ms: 190 }], 5) }, + { name: "wail", desc: "continuous rise/fall siren", make: () => + cat(sweep(500, 1200, 260, 16), sweep(1200, 500, 260, 16), + sweep(500, 1200, 260, 14)) }, + { name: "heartbeat", desc: "lub-dub x2", make: () => + [{ freq: 160, duty: 70, ms: 70 }, rest(40), { freq: 150, duty: 70, ms: 90 }, + rest(320), + { freq: 160, duty: 70, ms: 70 }, rest(40), { freq: 150, duty: 70, ms: 90 }] }, + + // --- character / fun --- + { name: "sadtrombone", desc: "wah-wah-wah-waaah", make: () => + cat(sweep(hz("Bb4"), hz("A4"), 260, 8, 55), [rest(30)], + sweep(hz("A4"), hz("Ab4"), 260, 8, 55), [rest(30)], + sweep(hz("Ab4"), hz("G4"), 260, 8, 55), [rest(30)], + sweep(hz("G4"), hz("Gb4"), 520, 14, 55)) }, + { name: "r2d2", desc: "warbling astromech chatter", make: () => + cat(sweep(900, 1900, 90, 8), sweep(1600, 700, 70, 6), + [rest(30)], sweep(1100, 2200, 80, 7), sweep(2000, 1200, 60, 5), + [rest(30)], sweep(800, 1500, 70, 6), sweep(1400, 2400, 90, 8)) }, + { name: "questcomplete", desc: "Zelda secret-found jingle", make: () => + notes("G5/150 F#5/150 D#5/150 A4/150 G#4/150 E5/150 G#5/150 C6/500") }, + + // --- melodies (multi-phrase) --- + { name: "nokia", desc: "Nokia tune (Gran Vals)", make: () => [ + notes("E6/120 D6/120 F#5/240 G#5/240"), + notes("C#6/120 B5/120 D5/240 E5/240"), + notes("B5/120 A5/120 C#5/240 E5/240"), + notes("A5/480"), + ] }, + { name: "imperial", desc: "Imperial March opening", make: () => [ + notes("G4/300 G4/300 G4/300 Eb4/210 Bb4/90"), + notes("G4/300 Eb4/210 Bb4/90 G4/600"), + ] }, + { name: "mario", desc: "Super Mario Bros. opening bars", make: () => [ + notes("E6/110 E6/110 ./110 E6/110 ./110 C6/110 E6/110"), + notes("G6/110 ./330 G5/110 ./330"), + notes("C6/150 ./75 G5/150 ./75 E5/150 ./75"), + notes("A5/110 B5/110 Bb5/110 A5/150"), + ] }, + { name: "scale", desc: "C-major run up an octave", make: () => + notes("C5/90 D5/90 E5/90 F5/90 G5/90 A5/90 B5/90 C6/180") }, +]; diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts index fb03cda8..a069ef7a 100644 --- a/app/ui/dashboard.ts +++ b/app/ui/dashboard.ts @@ -37,7 +37,7 @@ const dashboardActions: LayerActions = { setTranscribeRenderActive: () => {}, startVoiceCapture: () => {}, stopVoiceCapture: () => {}, - playBuzzerNote: () => {}, + playBuzzerSequence: () => {}, }; export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 }; diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 123f01b5..21ba6dc1 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -23,7 +23,8 @@ export type LayerActions = { startVoiceCapture: () => Promise | void; /** Stop voice capture; for a cloud provider this also commits for a final result. */ stopVoiceCapture: () => Promise | void; - playBuzzerNote: (note: number, oct: number, beat: number) => Promise | void; + /** Play a CFW tone-sequencer payload (see sound-effects.ts). */ + playBuzzerSequence: (payload: Uint8Array) => Promise | void; }; export type PaintBelow = () => GrayImage; diff --git a/app/ui/shell/music-app.ts b/app/ui/shell/music-app.ts new file mode 100644 index 00000000..56c1577b --- /dev/null +++ b/app/ui/shell/music-app.ts @@ -0,0 +1,243 @@ +import { getDefaultSmallFont } from "../../graphics/bdffont"; +import { GrayImage } from "../../graphics/image"; +import { wrapText } from "../../graphics/textwrap"; +import { clamp } from "../../util/numeric-util"; +import { + mediaControllerBridge, + type MediaControllerState, + type MediaQueueItem, +} from "../../native/media-controller"; +import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "./in-process-window"; + +export const MUSIC_WINDOW_ID = "music"; +export const MUSIC_SURFACE_ID = "window:music"; + +const ART_SIZE = 96; +const ART_X = 22; +const ART_Y = 40; +const META_X = ART_X + ART_SIZE + 14; +const LIST_TOP = 150; +const ROW_HEIGHT = 16; +const LIST_X = 22; +const FOOTER_HEIGHT = 20; + +type MusicRow = + | { kind: "play-pause"; label: string; enabled: boolean } + | { kind: "previous"; label: string; enabled: boolean } + | { kind: "next"; label: string; enabled: boolean } + | { kind: "queue-item"; label: string; enabled: true; item: MediaQueueItem }; + +/** + * Music controller app: metadata + album art + transport controls for the + * active Android media session (any player that publishes one), plus the + * player's queue when it exposes one (scroll to a track, click to jump). + */ +class MusicAppLayer implements Layer { + private selectedIndex = 0; + private scrollRow = 0; + private art: GrayImage | null = null; + private artKey = ""; + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + const media = mediaControllerBridge.snapshot(); + + image.drawText(font, 20, 8, "Music", 220); + const status = playbackLabel(media); + image.drawText(font, width - 24 - font.measureText(status), 8, status, 140); + + if (!media.accessEnabled) { + const lines = wrapText( + font, + "Notification access is required before Android exposes media sessions. Click to open settings.", + width - 48, + ); + for (let index = 0; index < lines.length; index++) { + image.drawText(font, 24, 44 + index * 14, lines[index]!, 180); + } + image.drawText(font, 20, height - 16, "Click: open settings Double-click: back", 110); + return image; + } + + if (!media.available) { + image.drawText(font, 24, 44, "No active media session.", 180); + image.drawText(font, 24, 62, "Start playback in another app on the phone.", 150); + image.drawText(font, 20, height - 16, "Double-click: back to sidebar", 110); + return image; + } + + this.drawArt(image, media); + + const metaWidth = width - META_X - 24; + const titleLines = wrapText(font, media.title || "Unknown title", metaWidth).slice(0, 2); + for (let index = 0; index < titleLines.length; index++) { + image.drawText(font, META_X, ART_Y + 2 + index * 15, titleLines[index]!, 230); + } + image.drawText(font, META_X, ART_Y + 36, media.artist || "Unknown artist", 180); + if (media.album) { + image.drawText(font, META_X, ART_Y + 52, truncate(font, media.album, metaWidth), 150); + } + image.drawText(font, META_X, ART_Y + 70, media.packageName, 110); + + const rows = this.buildRows(media); + this.selectedIndex = clamp(this.selectedIndex, 0, Math.max(0, rows.length - 1)); + const listHeight = height - LIST_TOP - FOOTER_HEIGHT; + const visibleRows = Math.max(1, (listHeight / ROW_HEIGHT) | 0); + if (this.selectedIndex < this.scrollRow) { + this.scrollRow = this.selectedIndex; + } else if (this.selectedIndex >= this.scrollRow + visibleRows) { + this.scrollRow = this.selectedIndex - visibleRows + 1; + } + this.scrollRow = clamp(this.scrollRow, 0, Math.max(0, rows.length - visibleRows)); + + const lastVisible = Math.min(rows.length, this.scrollRow + visibleRows); + for (let index = this.scrollRow; index < lastVisible; index++) { + const row = rows[index]!; + const y = LIST_TOP + (index - this.scrollRow) * ROW_HEIGHT; + const selected = index === this.selectedIndex; + if (selected) { + image.fillRoundedRect(LIST_X - 6, y - 1, width - 2 * LIST_X + 12, ROW_HEIGHT - 1, 15, 4); + image.drawRoundedRect(LIST_X - 6, y - 1, width - 2 * LIST_X + 12, ROW_HEIGHT - 1, 45, 4); + } + const value = !row.enabled ? (selected ? 130 : 90) : selected ? 255 : 200; + image.drawText(font, LIST_X, y + 1, truncate(font, row.label, width - 2 * LIST_X), value); + } + + image.drawText(font, 20, height - 16, "Scroll: select Click: activate Double-click: back", 110); + return image; + } + + async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { + const media = mediaControllerBridge.snapshot(); + if (!media.accessEnabled) { + if (event.type === "click") { + mediaControllerBridge.openNotificationAccessSettings(); + } else if (event.type === "double-click") { + ctx.stack.pop(); + } + return; + } + const rows = this.buildRows(media); + switch (event.type) { + case "scroll-up": + this.selectedIndex = Math.max(0, this.selectedIndex - 1); + return; + case "scroll-down": + this.selectedIndex = Math.min(Math.max(0, rows.length - 1), this.selectedIndex + 1); + return; + case "click": { + const row = rows[clamp(this.selectedIndex, 0, Math.max(0, rows.length - 1))]; + if (!row || !row.enabled) return; + if (row.kind === "play-pause") await mediaControllerBridge.playPause(); + else if (row.kind === "previous") await mediaControllerBridge.skipPrevious(); + else if (row.kind === "next") await mediaControllerBridge.skipNext(); + else if (row.kind === "queue-item") await mediaControllerBridge.skipToQueueItem(row.item.id); + return; + } + case "double-click": + ctx.stack.pop(); + return; + default: + return; + } + } + + private buildRows(media: MediaControllerState): MusicRow[] { + const rows: MusicRow[] = [ + { + kind: "play-pause", + label: media.playbackState === "playing" ? "Pause" : "Play", + enabled: media.canPlayPause, + }, + { kind: "previous", label: "Previous track", enabled: media.canSkipPrevious }, + { kind: "next", label: "Next track", enabled: media.canSkipNext }, + ]; + for (const item of mediaControllerBridge.getQueue()) { + rows.push({ + kind: "queue-item", + label: `${item.active ? "> " : " "}${item.title || "(untitled)"}`, + enabled: true, + item, + }); + } + return rows; + } + + private drawArt(image: GrayImage, media: MediaControllerState): void { + const key = `${media.packageName}|${media.title}|${media.album}`; + if (key !== this.artKey) { + this.artKey = key; + this.art = mediaControllerBridge.getAlbumArt(ART_SIZE); + } + if (this.art) { + // Center within the art box. + const dx = ART_X + Math.max(0, ((ART_SIZE - this.art.width) / 2) | 0); + const dy = ART_Y + Math.max(0, ((ART_SIZE - this.art.height) / 2) | 0); + image.bitBlt(this.art, dx, dy); + image.drawRect(dx - 1, dy - 1, this.art.width + 2, this.art.height + 2, 60); + } else { + image.drawRect(ART_X, ART_Y, ART_SIZE, ART_SIZE, 60); + const font = getDefaultSmallFont(); + image.drawText(font, ART_X + 22, ART_Y + ART_SIZE / 2 - 7, "no art", 90); + } + } +} + +export function createMusicAppWindow(options: InProcessAppOptions): InProcessWindow { + let unsubscribe: (() => void) | null = null; + const app = createInProcessWindow({ + appId: "music", + windowId: MUSIC_WINDOW_ID, + title: "Music", + iconLetter: "M", + closeable: true, + actions: options.actions, + baseLayer: new YieldAtRootLayer(new MusicAppLayer()), + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + onClosed: () => { + unsubscribe?.(); + unsubscribe = null; + options.onClosed(); + }, + }); + unsubscribe = mediaControllerBridge.onStateChange(() => { + app.requestRender(); + }); + return app; +} + +function playbackLabel(media: MediaControllerState): string { + switch (media.playbackState) { + case "playing": + return "Playing"; + case "paused": + return "Paused"; + case "buffering": + return "Buffering"; + case "stopped": + return "Stopped"; + case "notification-access-required": + return "Access required"; + default: + return media.status || "Idle"; + } +} + +function truncate(font: import("../../graphics/bdffont").BdfFont, text: string, maxWidth: number): string { + if (font.measureText(text) <= maxWidth) return text; + let out = text; + while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) { + out = out.slice(0, -1); + } + return `${out}...`; +} diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 7d5338ca..185b0bac 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -69,7 +69,7 @@ const noopActions: LayerActions = { setTranscribeRenderActive: () => {}, startVoiceCapture: () => {}, stopVoiceCapture: () => {}, - playBuzzerNote: () => {}, + playBuzzerSequence: () => {}, }; /** Long-press overlay menu; closing it returns focus to the sidebar. */