forked from Blendman974/faceclaw
Work on voice input
This commit is contained in:
1 parent
7b8fc8f37f
commit
097d40efb2
12 files changed
+599
-234
No files matched your search
+29
-49
@@ -1,37 +1,37 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
export class TranscribeLayer implements Layer {
|
||||
private mode: "off" | "wakeword" | "full" = "full";
|
||||
private status = "Starting transcription...";
|
||||
private transcript = "";
|
||||
// Finalized utterances, plus the live (replace-semantics) partial appended
|
||||
// when painting.
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private actions: LayerActions | null = null;
|
||||
|
||||
start(ctx: LayerContext): void {
|
||||
this.actions = ctx.actions;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
void ctx.actions.setTranscribeRenderActive(true);
|
||||
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
|
||||
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
|
||||
});
|
||||
void Promise.resolve(ctx.actions.startVoiceCapture());
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
paint(): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
|
||||
const text = this.transcript || "Listening...";
|
||||
const text = this.displayText() || "Listening...";
|
||||
const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64);
|
||||
|
||||
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
|
||||
image.drawText(font, 24, 24, "Transcribe", 200);
|
||||
image.drawText(font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
|
||||
//image.drawText(font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
|
||||
image.drawText(font, 24, 46, this.status, 110);
|
||||
|
||||
const firstLine = Math.max(0, wrapped.length - 10);
|
||||
@@ -40,63 +40,46 @@ export class TranscribeLayer implements Layer {
|
||||
image.drawText(font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(font, 24, 252, "Click: mode / Double-click: back", 110);
|
||||
image.drawText(font, 24, 252, "Double-click: back", 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
this.stop(ctx);
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click") {
|
||||
this.cycleMode(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
private cycleMode(ctx: LayerContext): void {
|
||||
if (this.mode === "full") {
|
||||
this.mode = "wakeword";
|
||||
void ctx.actions.startDedicatedVoiceInput("wakeword");
|
||||
} else if (this.mode === "wakeword") {
|
||||
this.mode = "off";
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
} else {
|
||||
this.mode = "full";
|
||||
void ctx.actions.startDedicatedVoiceInput("full");
|
||||
}
|
||||
}
|
||||
|
||||
private stop(ctx: LayerContext): void {
|
||||
onRemoved(): void {
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
void ctx.actions.setTranscribeRenderActive(false);
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
void this.actions?.setTranscribeRenderActive(false);
|
||||
void this.actions?.stopVoiceCapture();
|
||||
this.actions = null;
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
if (this.transcript && !this.transcript.endsWith("\n")) {
|
||||
this.transcript = `${this.transcript}\n`;
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
return;
|
||||
this.liveText = "";
|
||||
} else {
|
||||
// Replace semantics: the live partial is the whole current best transcript.
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
this.transcript = appendTranscriptChunk(this.transcript, event.text);
|
||||
}
|
||||
}
|
||||
|
||||
function appendTranscriptChunk(transcript: string, chunk: string): string {
|
||||
const trimmed = chunk.replace(/\s+/g, " ").trim();
|
||||
if (!trimmed) return transcript;
|
||||
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
|
||||
return `${transcript}${trimmed}`;
|
||||
}
|
||||
return `${transcript} ${trimmed}`;
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
@@ -116,6 +99,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncateTranscribeLine(text: string, maxChars: number): string {
|
||||
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
const DIALOG_X = 40;
|
||||
const DIALOG_Y = 40;
|
||||
const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
|
||||
/**
|
||||
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
|
||||
* The mic runs while the button is held (long-press); releasing it stops the
|
||||
* mic and finalizes. The dialog stays up until dismissed with a double-click.
|
||||
*/
|
||||
export class VoiceInputLayer implements Layer {
|
||||
private status = "Listening...";
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private capturing = false;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
) {}
|
||||
|
||||
startCapture(): void {
|
||||
if (this.capturing) return;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.actions.requestRender();
|
||||
});
|
||||
this.capturing = true;
|
||||
void this.actions.startVoiceCapture();
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Button released: stop the mic and finalize, but keep the dialog open. */
|
||||
endCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.status.startsWith("Listening")) {
|
||||
this.status = "Double-click to close.";
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
// Solid dialog box over the underlying UI.
|
||||
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10);
|
||||
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
|
||||
|
||||
const left = DIALOG_X + 16;
|
||||
image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220);
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
|
||||
|
||||
const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)");
|
||||
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
|
||||
const maxLines = 8;
|
||||
const firstLine = Math.max(0, wrapped.length - maxLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
if (this.capturing) {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
}
|
||||
this.onClosed();
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
} else {
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) lines.push(line);
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncate(font: BdfFont, text: string, maxWidth: number): string {
|
||||
if (font.measureText(text) <= maxWidth) return text;
|
||||
let out = text;
|
||||
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
|
||||
out = out.slice(0, -1);
|
||||
}
|
||||
return `${out}...`;
|
||||
}
|
||||
Reference in new issue
Block a user