forked from Blendman974/faceclaw
Add Transcribe screen
This commit is contained in:
1 parent
811477811e
commit
bcc30545ad
16 files changed
+904
-20
No files matched your search
@@ -52,6 +52,7 @@ const CONTAINER_NAME = "dashboard";
|
||||
const DASHBOARD_INTERVAL_MS = 60_000;
|
||||
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
|
||||
const STOPWATCH_RENDER_INTERVAL_MS = 100;
|
||||
const TRANSCRIBE_RENDER_INTERVAL_MS = 250;
|
||||
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
|
||||
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
|
||||
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
|
||||
@@ -120,6 +121,7 @@ class DashboardController {
|
||||
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private stopwatchRenderTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private transcribeRenderTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private offState: (() => void) | null = null;
|
||||
private offLog: (() => void) | null = null;
|
||||
private offRing: (() => void) | null = null;
|
||||
@@ -149,6 +151,9 @@ class DashboardController {
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
|
||||
setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active),
|
||||
setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active),
|
||||
startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode),
|
||||
stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -393,6 +398,10 @@ class DashboardController {
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
@@ -452,6 +461,10 @@ class DashboardController {
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
await communicator?.close().catch(() => {});
|
||||
} finally {
|
||||
stopForegroundNotification();
|
||||
@@ -673,7 +686,21 @@ class DashboardController {
|
||||
if (!this.communicator) return;
|
||||
if (!isDashboardVoiceControlEnabled()) return;
|
||||
this.updateConnectedForegroundNotification();
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator());
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword");
|
||||
}
|
||||
|
||||
private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise<void> {
|
||||
if (!this.communicator) {
|
||||
throw new Error("Voice input needs an active G2 connection.");
|
||||
}
|
||||
await ensureVoicePermissions();
|
||||
this.updateConnectedForegroundNotification();
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode);
|
||||
}
|
||||
|
||||
private stopDedicatedVoiceInput(): void {
|
||||
voiceControlBridge.stop();
|
||||
this.startVoiceControlIfEnabled();
|
||||
}
|
||||
|
||||
private setStopwatchRenderActive(active: boolean): void {
|
||||
@@ -692,6 +719,22 @@ class DashboardController {
|
||||
}, STOPWATCH_RENDER_INTERVAL_MS);
|
||||
}
|
||||
|
||||
private setTranscribeRenderActive(active: boolean): void {
|
||||
if (!active || this.phase !== "connected" || !this.communicator) {
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (this.transcribeRenderTimer) return;
|
||||
this.transcribeRenderTimer = setInterval(() => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`transcribe render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}, TRANSCRIBE_RENDER_INTERVAL_MS);
|
||||
}
|
||||
|
||||
private clearDashboardTimer(): void {
|
||||
if (this.dashboardTimer) {
|
||||
clearInterval(this.dashboardTimer);
|
||||
@@ -705,6 +748,10 @@ class DashboardController {
|
||||
clearInterval(this.stopwatchRenderTimer);
|
||||
this.stopwatchRenderTimer = null;
|
||||
}
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private setPhase(phase: ConnectionPhase): void {
|
||||
|
||||
@@ -6,13 +6,21 @@ export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type VoiceInputMode = "off" | "wakeword" | "full";
|
||||
export type VoiceTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private started = false;
|
||||
private mode: VoiceInputMode = "off";
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -25,19 +33,34 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.wakeWordListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(communicator?: any): void {
|
||||
if (this.started || !global.isAndroid) return;
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcriptListeners.add(listener);
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(communicator?: any, mode: VoiceInputMode = "wakeword"): void {
|
||||
if (mode === "off") {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
if (!global.isAndroid) return;
|
||||
this.ensureController();
|
||||
if (communicator) {
|
||||
this.controller?.setCommunicator(communicator);
|
||||
}
|
||||
if (this.started && this.mode !== mode) {
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
}
|
||||
this.started = true;
|
||||
this.controller?.start();
|
||||
this.mode = mode;
|
||||
this.controller?.start(mode);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started || !global.isAndroid) return;
|
||||
this.started = false;
|
||||
this.mode = "off";
|
||||
this.controller?.stop();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
@@ -58,6 +81,12 @@ export class FaceclawVoiceControlBridge {
|
||||
listener(String(keyword));
|
||||
}
|
||||
},
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
const event = { text: String(text), isFinal: Boolean(isFinal) };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { nightscoutBridge } from "../native/nightscout";
|
||||
import { readActiveNotificationIcons } from "../native/notification-icons";
|
||||
import { readPhoneBatteryState } from "../native/phone-battery";
|
||||
import { readSystemStatusIcons } from "../native/system-status-icons";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../native/voice-control";
|
||||
import { EventSourceType, OsEventTypeList } from "../g2/events";
|
||||
import {
|
||||
getDashboardPlugin,
|
||||
@@ -85,6 +86,9 @@ const dashboardActions: LayerActions = {
|
||||
endTextSettingEdit: () => {},
|
||||
setVoiceControlEnabled: () => {},
|
||||
setStopwatchRenderActive: () => {},
|
||||
setTranscribeRenderActive: () => {},
|
||||
startDedicatedVoiceInput: () => {},
|
||||
stopDedicatedVoiceInput: () => {},
|
||||
};
|
||||
const dashboardFont = loadEmbeddedTerminus12();
|
||||
const dashboardSystemFont = loadEmbeddedTerminus16();
|
||||
@@ -487,6 +491,14 @@ function createAppsMenuLayer(): MenuLayer {
|
||||
void ctx.actions.setStopwatchRenderActive(true);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
const layer = new TranscribeLayer();
|
||||
ctx.stack.push(layer);
|
||||
layer.start(ctx);
|
||||
},
|
||||
},
|
||||
],
|
||||
TOP_LEFT_MENU_LAYOUT,
|
||||
);
|
||||
@@ -539,6 +551,122 @@ class StopwatchLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
class TranscribeLayer implements Layer {
|
||||
private mode: "off" | "wakeword" | "full" = "full";
|
||||
private status = "Starting transcription...";
|
||||
private transcript = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
start(ctx: LayerContext): void {
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
});
|
||||
void ctx.actions.setTranscribeRenderActive(true);
|
||||
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
|
||||
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
|
||||
});
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
|
||||
const text = this.transcript || "Listening...";
|
||||
const wrapped = wrapTranscribeText(ctx.font, text, G2_LENS_WIDTH - 64);
|
||||
|
||||
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
|
||||
image.drawText(ctx.font, 24, 24, "Transcribe", 200);
|
||||
image.drawText(ctx.font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
|
||||
//image.drawText(ctx.font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
|
||||
image.drawText(ctx.font, 24, 46, this.status, 110);
|
||||
|
||||
const firstLine = Math.max(0, wrapped.length - 10);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
const y = 72 + (index - firstLine) * 16;
|
||||
image.drawText(ctx.font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(ctx.font, 24, 252, "Click: mode / Double-click: back", 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
this.stop(ctx);
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click") {
|
||||
this.cycleMode(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
private cycleMode(ctx: LayerContext): void {
|
||||
if (this.mode === "full") {
|
||||
this.mode = "wakeword";
|
||||
void ctx.actions.startDedicatedVoiceInput("wakeword");
|
||||
} else if (this.mode === "wakeword") {
|
||||
this.mode = "off";
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
} else {
|
||||
this.mode = "full";
|
||||
void ctx.actions.startDedicatedVoiceInput("full");
|
||||
}
|
||||
}
|
||||
|
||||
private stop(ctx: LayerContext): void {
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
void ctx.actions.setTranscribeRenderActive(false);
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
if (this.transcript && !this.transcript.endsWith("\n")) {
|
||||
this.transcript = `${this.transcript}\n`;
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.transcript = appendTranscriptChunk(this.transcript, event.text);
|
||||
}
|
||||
}
|
||||
|
||||
function appendTranscriptChunk(transcript: string, chunk: string): string {
|
||||
const trimmed = chunk.replace(/\s+/g, " ").trim();
|
||||
if (!trimmed) return transcript;
|
||||
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
|
||||
return `${transcript}${trimmed}`;
|
||||
}
|
||||
return `${transcript} ${trimmed}`;
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: typeof dashboardFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncateTranscribeLine(text: string, maxChars: number): string {
|
||||
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
|
||||
}
|
||||
|
||||
function formatStopwatchElapsed(elapsedMs: number): string {
|
||||
const totalTenths = Math.floor(elapsedMs / 100);
|
||||
const tenths = totalTenths % 10;
|
||||
|
||||
@@ -17,6 +17,9 @@ export type LayerActions = {
|
||||
endTextSettingEdit: () => Promise<void> | void;
|
||||
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
|
||||
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
|
||||
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
|
||||
startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise<void> | void;
|
||||
stopDedicatedVoiceInput: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type PaintBelow = () => GrayImage;
|
||||
|
||||
Reference in new issue
Block a user