Work on voice input

This commit is contained in:
James Babcock committed 2026-07-05 20:59:53 -07:00
1 parent 7b8fc8f37f
commit 097d40efb2
12 files changed
+599 -234

No files matched your search

+29 -49
View File
@@ -1,37 +1,37 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
export class TranscribeLayer implements Layer {
private mode: "off" | "wakeword" | "full" = "full";
private status = "Starting transcription...";
private transcript = "";
// Finalized utterances, plus the live (replace-semantics) partial appended
// when painting.
private finalizedText = "";
private liveText = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private actions: LayerActions | null = null;
start(ctx: LayerContext): void {
this.actions = ctx.actions;
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
ctx.actions.requestRender();
});
void ctx.actions.setTranscribeRenderActive(true);
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
});
void Promise.resolve(ctx.actions.startVoiceCapture());
}
paint(ctx: LayerContext): GrayImage {
paint(): GrayImage {
const font = getDefaultSmallFont();
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
const text = this.transcript || "Listening...";
const text = this.displayText() || "Listening...";
const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64);
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
image.drawText(font, 24, 24, "Transcribe", 200);
image.drawText(font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
//image.drawText(font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
image.drawText(font, 24, 46, this.status, 110);
const firstLine = Math.max(0, wrapped.length - 10);
@@ -40,63 +40,46 @@ export class TranscribeLayer implements Layer {
image.drawText(font, 32, y, wrapped[index]!, 230);
}
image.drawText(font, 24, 252, "Click: mode / Double-click: back", 110);
image.drawText(font, 24, 252, "Double-click: back", 110);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
this.stop(ctx);
ctx.stack.pop();
return;
}
if (event.type === "click") {
this.cycleMode(ctx);
}
}
private cycleMode(ctx: LayerContext): void {
if (this.mode === "full") {
this.mode = "wakeword";
void ctx.actions.startDedicatedVoiceInput("wakeword");
} else if (this.mode === "wakeword") {
this.mode = "off";
void ctx.actions.stopDedicatedVoiceInput();
} else {
this.mode = "full";
void ctx.actions.startDedicatedVoiceInput("full");
}
}
private stop(ctx: LayerContext): void {
onRemoved(): void {
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
void ctx.actions.setTranscribeRenderActive(false);
void ctx.actions.stopDedicatedVoiceInput();
void this.actions?.setTranscribeRenderActive(false);
void this.actions?.stopVoiceCapture();
this.actions = null;
}
private displayText(): string {
if (!this.finalizedText) return this.liveText;
if (!this.liveText) return this.finalizedText;
return `${this.finalizedText} ${this.liveText}`;
}
private onTranscript(event: VoiceTranscriptEvent): void {
if (event.isFinal) {
if (this.transcript && !this.transcript.endsWith("\n")) {
this.transcript = `${this.transcript}\n`;
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
return;
this.liveText = "";
} else {
// Replace semantics: the live partial is the whole current best transcript.
this.liveText = event.text.trim();
}
this.transcript = appendTranscriptChunk(this.transcript, event.text);
}
}
function appendTranscriptChunk(transcript: string, chunk: string): string {
const trimmed = chunk.replace(/\s+/g, " ").trim();
if (!trimmed) return transcript;
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
return `${transcript}${trimmed}`;
}
return `${transcript} ${trimmed}`;
}
function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
@@ -116,6 +99,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri
return lines.length ? lines : [""];
}
function truncateTranscribeLine(text: string, maxChars: number): string {
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
}
+137
View File
@@ -0,0 +1,137 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
const DIALOG_X = 40;
const DIALOG_Y = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
/**
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
* The mic runs while the button is held (long-press); releasing it stops the
* mic and finalizes. The dialog stays up until dismissed with a double-click.
*/
export class VoiceInputLayer implements Layer {
private status = "Listening...";
private finalizedText = "";
private liveText = "";
private capturing = false;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
) {}
startCapture(): void {
if (this.capturing) return;
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
this.actions.requestRender();
});
this.capturing = true;
void this.actions.startVoiceCapture();
this.actions.requestRender();
}
/** Button released: stop the mic and finalize, but keep the dialog open. */
endCapture(): void {
if (!this.capturing) return;
this.capturing = false;
void this.actions.stopVoiceCapture();
if (this.status.startsWith("Listening")) {
this.status = "Double-click to close.";
}
this.actions.requestRender();
}
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
// Solid dialog box over the underlying UI.
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10);
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220);
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)");
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
const maxLines = 8;
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235);
}
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
ctx.stack.pop();
}
}
onRemoved(): void {
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
if (this.capturing) {
this.capturing = false;
void this.actions.stopVoiceCapture();
}
this.onClosed();
}
private displayText(): string {
if (!this.finalizedText) return this.liveText;
if (!this.liveText) return this.finalizedText;
return `${this.finalizedText} ${this.liveText}`;
}
private onTranscript(event: VoiceTranscriptEvent): void {
if (event.isFinal) {
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
this.liveText = "";
} else {
this.liveText = event.text.trim();
}
this.actions.requestRender();
}
}
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}
function truncate(font: BdfFont, text: string, maxWidth: number): string {
if (font.measureText(text) <= maxWidth) return text;
let out = text;
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
out = out.slice(0, -1);
}
return `${out}...`;
}
+25 -2
View File
@@ -267,6 +267,27 @@ export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
defaultValue: false,
});
export type VoiceProvider = "onboard" | "elevenlabs";
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
id: "voice-provider",
label: "Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: ["onboard", "elevenlabs"],
formatValue: (value) => (value === "elevenlabs" ? "ElevenLabs" : "On-device"),
});
export const elevenLabsApiKeySetting = new ConfigSettingString({
id: "elevenlabs-api-key",
label: "ElevenLabs key",
storageKey: "voice.elevenLabsApiKey",
defaultValue: "",
editorTitle: "ElevenLabs API key",
glassesEditTitle: "Edit ElevenLabs key",
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
});
export const terminalHostSetting = new ConfigSettingString({
id: "terminal-host",
label: "Host",
@@ -448,8 +469,10 @@ export function textSettingMenuItem<TId extends string = string>(
void ctx.actions.startTextSettingEdit(setting);
ctx.stack.push(new EditTextSettingLayer(setting));
},
render: ({ image, x, y, width, selected }) => {
image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.get())}`, 200);
render: ({ image, x, y }) => {
// displayValue honors the setting's formatValue, so secrets (API keys,
// tokens) can mask themselves instead of rendering in the clear.
image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.displayValue())}`, 200);
}
};
}
+25 -5
View File
@@ -1,5 +1,4 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { getDefaultSmallFont } from "../graphics/bdffont";
import { type RawInputEvent } from "../native/faceclaw-communicator";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
@@ -9,6 +8,7 @@ import { EditTextSettingLayer, isNightscoutSettingsConfigured, screenTimeoutSett
import { Layer, LayerActions, LayerStack, type DashboardInputEvent, type LayerContext } from "./layers";
import { SingleNotificationLayer } from "./notifications";
import { TelepromptLayer } from "./apps/teleprompt";
import { VoiceInputLayer } from "./apps/voice-input";
import { drawSystemCard } from "./dashboard/system-card";
import { createRootMenuLayer } from "./dashboard/root-menu";
@@ -44,16 +44,19 @@ const dashboardActions: LayerActions = {
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
setVoiceControlEnabled: () => {},
setStopwatchRenderActive: () => {},
setTranscribeRenderActive: () => {},
startDedicatedVoiceInput: () => {},
stopDedicatedVoiceInput: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
playBuzzerNote: () => {},
};
const dashboardFont = getDefaultSmallFont();
export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 };
// The push-to-talk dialog is a singleton on the layer stack; tracked here so
// long-press-release (a global event) can reach it and so a second long-press
// doesn't stack duplicates.
let activeVoiceInputLayer: VoiceInputLayer | null = null;
function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
@@ -117,6 +120,23 @@ export async function receiveInput(event: RawInputEvent): Promise<void> {
}
return;
}
// Push-to-talk is intercepted globally so it works over any UI: long-press
// opens the voice dialog and starts the mic; releasing it stops the mic.
if (inputEvent.type === "long-press") {
if (!activeVoiceInputLayer) {
const layer = new VoiceInputLayer(dashboardActions, () => {
if (activeVoiceInputLayer === layer) activeVoiceInputLayer = null;
});
activeVoiceInputLayer = layer;
dashboardLayers.push(layer);
layer.startCapture();
}
return;
}
if (inputEvent.type === "long-press-release") {
activeVoiceInputLayer?.endCapture();
return;
}
await dashboardLayers.handleInput(inputEvent);
}
+3 -2
View File
@@ -1,5 +1,5 @@
import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { MenuLayer } from "../menu";
export function createSettingsMenuLayer(): MenuLayer {
@@ -69,7 +69,8 @@ function createVoiceSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings > Voice",
[
toggleSettingMenuItem(voiceControlEnabledSetting),
enumSettingMenuItem(voiceProviderSetting),
textSettingMenuItem(elevenLabsApiKeySetting),
],
TOP_LEFT_MENU_LAYOUT,
);
+4 -3
View File
@@ -18,11 +18,12 @@ export type LayerActions = {
disconnect: () => Promise<void> | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise<void> | void;
stopDedicatedVoiceInput: () => Promise<void> | void;
/** Start voice capture with the provider chosen in settings. */
startVoiceCapture: () => Promise<void> | void;
/** Stop voice capture; for a cloud provider this also commits for a final result. */
stopVoiceCapture: () => Promise<void> | void;
playBuzzerNote: (note: number, oct: number, beat: number) => Promise<void> | void;
};