Add Transcribe screen

This commit is contained in:
James Babcock committed 2026-05-01 15:00:53 -07:00
1 parent 811477811e
commit bcc30545ad
16 files changed
+904 -20

No files matched your search

+48 -1
View File
@@ -52,6 +52,7 @@ const CONTAINER_NAME = "dashboard";
const DASHBOARD_INTERVAL_MS = 60_000;
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
const STOPWATCH_RENDER_INTERVAL_MS = 100;
const TRANSCRIBE_RENDER_INTERVAL_MS = 250;
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
@@ -120,6 +121,7 @@ class DashboardController {
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
private stopwatchRenderTimer: ReturnType<typeof setInterval> | null = null;
private transcribeRenderTimer: ReturnType<typeof setInterval> | null = null;
private offState: (() => void) | null = null;
private offLog: (() => void) | null = null;
private offRing: (() => void) | null = null;
@@ -149,6 +151,9 @@ class DashboardController {
endTextSettingEdit: () => this.endTextSettingEdit(),
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active),
setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active),
startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode),
stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(),
});
}
@@ -393,6 +398,10 @@ class DashboardController {
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
@@ -452,6 +461,10 @@ class DashboardController {
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
await communicator?.close().catch(() => {});
} finally {
stopForegroundNotification();
@@ -673,7 +686,21 @@ class DashboardController {
if (!this.communicator) return;
if (!isDashboardVoiceControlEnabled()) return;
this.updateConnectedForegroundNotification();
voiceControlBridge.start(this.communicator.getNativeCommunicator());
voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword");
}
private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise<void> {
if (!this.communicator) {
throw new Error("Voice input needs an active G2 connection.");
}
await ensureVoicePermissions();
this.updateConnectedForegroundNotification();
voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode);
}
private stopDedicatedVoiceInput(): void {
voiceControlBridge.stop();
this.startVoiceControlIfEnabled();
}
private setStopwatchRenderActive(active: boolean): void {
@@ -692,6 +719,22 @@ class DashboardController {
}, STOPWATCH_RENDER_INTERVAL_MS);
}
private setTranscribeRenderActive(active: boolean): void {
if (!active || this.phase !== "connected" || !this.communicator) {
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
return;
}
if (this.transcribeRenderTimer) return;
this.transcribeRenderTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => {
this.appendLog(`transcribe render failed: ${this.formatError(error)}`);
});
}, TRANSCRIBE_RENDER_INTERVAL_MS);
}
private clearDashboardTimer(): void {
if (this.dashboardTimer) {
clearInterval(this.dashboardTimer);
@@ -705,6 +748,10 @@ class DashboardController {
clearInterval(this.stopwatchRenderTimer);
this.stopwatchRenderTimer = null;
}
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
}
private setPhase(phase: ConnectionPhase): void {
+32 -3
View File
@@ -6,13 +6,21 @@ export type VoiceControlState = {
status: string;
};
export type VoiceInputMode = "off" | "wakeword" | "full";
export type VoiceTranscriptEvent = {
text: string;
isFinal: boolean;
};
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
private started = false;
private mode: VoiceInputMode = "off";
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
@@ -25,19 +33,34 @@ export class FaceclawVoiceControlBridge {
return () => this.wakeWordListeners.delete(listener);
}
start(communicator?: any): void {
if (this.started || !global.isAndroid) return;
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
this.transcriptListeners.add(listener);
return () => this.transcriptListeners.delete(listener);
}
start(communicator?: any, mode: VoiceInputMode = "wakeword"): void {
if (mode === "off") {
this.stop();
return;
}
if (!global.isAndroid) return;
this.ensureController();
if (communicator) {
this.controller?.setCommunicator(communicator);
}
if (this.started && this.mode !== mode) {
this.controller?.stop();
this.started = false;
}
this.started = true;
this.controller?.start();
this.mode = mode;
this.controller?.start(mode);
}
stop(): void {
if (!this.started || !global.isAndroid) return;
this.started = false;
this.mode = "off";
this.controller?.stop();
this.setStatus("Voice control stopped.");
}
@@ -58,6 +81,12 @@ export class FaceclawVoiceControlBridge {
listener(String(keyword));
}
},
onTranscript: (text: string, isFinal: boolean) => {
const event = { text: String(text), isFinal: Boolean(isFinal) };
for (const listener of this.transcriptListeners) {
listener(event);
}
},
});
this.controller.setListener(this.listenerProxy);
}
+128
View File
@@ -8,6 +8,7 @@ import { nightscoutBridge } from "../native/nightscout";
import { readActiveNotificationIcons } from "../native/notification-icons";
import { readPhoneBatteryState } from "../native/phone-battery";
import { readSystemStatusIcons } from "../native/system-status-icons";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../native/voice-control";
import { EventSourceType, OsEventTypeList } from "../g2/events";
import {
getDashboardPlugin,
@@ -85,6 +86,9 @@ const dashboardActions: LayerActions = {
endTextSettingEdit: () => {},
setVoiceControlEnabled: () => {},
setStopwatchRenderActive: () => {},
setTranscribeRenderActive: () => {},
startDedicatedVoiceInput: () => {},
stopDedicatedVoiceInput: () => {},
};
const dashboardFont = loadEmbeddedTerminus12();
const dashboardSystemFont = loadEmbeddedTerminus16();
@@ -487,6 +491,14 @@ function createAppsMenuLayer(): MenuLayer {
void ctx.actions.setStopwatchRenderActive(true);
},
},
{
label: "Transcribe",
onSelect: (ctx) => {
const layer = new TranscribeLayer();
ctx.stack.push(layer);
layer.start(ctx);
},
},
],
TOP_LEFT_MENU_LAYOUT,
);
@@ -539,6 +551,122 @@ class StopwatchLayer implements Layer {
}
}
class TranscribeLayer implements Layer {
private mode: "off" | "wakeword" | "full" = "full";
private status = "Starting transcription...";
private transcript = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
start(ctx: LayerContext): void {
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
});
void ctx.actions.setTranscribeRenderActive(true);
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
});
}
paint(ctx: LayerContext): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
const text = this.transcript || "Listening...";
const wrapped = wrapTranscribeText(ctx.font, text, G2_LENS_WIDTH - 64);
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
image.drawText(ctx.font, 24, 24, "Transcribe", 200);
image.drawText(ctx.font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
//image.drawText(ctx.font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
image.drawText(ctx.font, 24, 46, this.status, 110);
const firstLine = Math.max(0, wrapped.length - 10);
for (let index = firstLine; index < wrapped.length; index++) {
const y = 72 + (index - firstLine) * 16;
image.drawText(ctx.font, 32, y, wrapped[index]!, 230);
}
image.drawText(ctx.font, 24, 252, "Click: mode / Double-click: back", 110);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
this.stop(ctx);
ctx.stack.pop();
return;
}
if (event.type === "click") {
this.cycleMode(ctx);
}
}
private cycleMode(ctx: LayerContext): void {
if (this.mode === "full") {
this.mode = "wakeword";
void ctx.actions.startDedicatedVoiceInput("wakeword");
} else if (this.mode === "wakeword") {
this.mode = "off";
void ctx.actions.stopDedicatedVoiceInput();
} else {
this.mode = "full";
void ctx.actions.startDedicatedVoiceInput("full");
}
}
private stop(ctx: LayerContext): void {
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
void ctx.actions.setTranscribeRenderActive(false);
void ctx.actions.stopDedicatedVoiceInput();
}
private onTranscript(event: VoiceTranscriptEvent): void {
if (event.isFinal) {
if (this.transcript && !this.transcript.endsWith("\n")) {
this.transcript = `${this.transcript}\n`;
}
return;
}
this.transcript = appendTranscriptChunk(this.transcript, event.text);
}
}
function appendTranscriptChunk(transcript: string, chunk: string): string {
const trimmed = chunk.replace(/\s+/g, " ").trim();
if (!trimmed) return transcript;
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
return `${transcript}${trimmed}`;
}
return `${transcript} ${trimmed}`;
}
function wrapTranscribeText(font: typeof dashboardFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) {
lines.push(line);
}
return lines.length ? lines : [""];
}
function truncateTranscribeLine(text: string, maxChars: number): string {
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
}
function formatStopwatchElapsed(elapsedMs: number): string {
const totalTenths = Math.floor(elapsedMs / 100);
const tenths = totalTenths % 10;
+3
View File
@@ -17,6 +17,9 @@ export type LayerActions = {
endTextSettingEdit: () => Promise<void> | void;
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise<void> | void;
stopDedicatedVoiceInput: () => Promise<void> | void;
};
export type PaintBelow = () => GrayImage;