forked from Blendman974/faceclaw
Improve the Transcribe app
This commit is contained in:
1 parent
c9a35b0922
commit
9b70880a5b
12 files changed
+540
-140
No files matched your search
@@ -57,10 +57,18 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
}],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
options.stopContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
options.onClosed();
|
||||
|
||||
@@ -1,33 +1,35 @@
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads on click.
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*/
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Listening...";
|
||||
// Finalized utterances, plus the live (replace-semantics) partial appended
|
||||
// when painting.
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.onTranscript(event);
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
});
|
||||
}
|
||||
@@ -36,29 +38,34 @@ export class TranscribeLayer implements Layer {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const text = this.displayText() || "Listening...";
|
||||
const wrapped = wrapTranscribeText(font, text, width - 64);
|
||||
const text = this.transcript.text || "Listening...";
|
||||
const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const footerY = height - font.lineHeight - 6;
|
||||
const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / step));
|
||||
const firstLine = Math.max(0, wrapped.length - bodyLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
const y = bodyTop + (index - firstLine) * step;
|
||||
image.drawText(font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(font, 24, footerY, `${GESTURE_CLICK} save ${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
if (wrapped.length > bodyLines) {
|
||||
const trackHeight = height - bodyTop - 6;
|
||||
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / wrapped.length));
|
||||
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (wrapped.length - bodyLines));
|
||||
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
|
||||
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "click") {
|
||||
this.saveTranscript();
|
||||
if (event.type === "scroll-up" || event.type === "scroll-down") {
|
||||
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
@@ -68,14 +75,16 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
}
|
||||
|
||||
private saveTranscript(): void {
|
||||
const text = this.displayText().trim();
|
||||
saveTranscript(): void {
|
||||
const text = this.transcript.text.trim();
|
||||
if (!text) {
|
||||
this.saveNotice = "Nothing to save yet.";
|
||||
return;
|
||||
@@ -84,25 +93,6 @@ export class TranscribeLayer implements Layer {
|
||||
const path = writeTextToDownloads(filename, `${text}\n`);
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
} else {
|
||||
// Replace semantics: the live partial is the whole current best transcript.
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptTimestamp(): string {
|
||||
@@ -113,22 +103,3 @@ function transcriptTimestamp(): string {
|
||||
`-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: UiFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
type TranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
transcribeText?: string;
|
||||
paragraphBreakAfter?: boolean;
|
||||
};
|
||||
|
||||
/** Text accumulation belongs to this app, not the shared dictation stream. */
|
||||
export class TranscriptModel {
|
||||
private finalized = "";
|
||||
private live = "";
|
||||
private separator = " ";
|
||||
private breakAfterLive = false;
|
||||
|
||||
get text(): string {
|
||||
return this.finalized && this.live ? this.finalized + this.separator + this.live : this.finalized || this.live;
|
||||
}
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
const text = (event.transcribeText ?? event.text).trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
if (finalText) this.finalized = this.finalized ? this.finalized + this.separator + finalText : finalText;
|
||||
this.live = "";
|
||||
if (finalText) this.separator = event.paragraphBreakAfter || this.breakAfterLive ? "\n" : " ";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
|
||||
/** The onboard recognizer may finalize just before or just after the pause. */
|
||||
pause(): void {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
export class TranscriptScroll {
|
||||
firstLine = 0;
|
||||
private maximum = 0;
|
||||
private following = true;
|
||||
|
||||
layout(totalLines: number, visibleLines: number): number {
|
||||
this.maximum = Math.max(0, totalLines - visibleLines);
|
||||
this.firstLine = this.following ? this.maximum : Math.min(this.firstLine, this.maximum);
|
||||
return this.firstLine;
|
||||
}
|
||||
|
||||
scroll(delta: number): void {
|
||||
this.firstLine = Math.max(0, Math.min(this.maximum, this.firstLine + delta));
|
||||
this.following = this.firstLine === this.maximum;
|
||||
}
|
||||
}
|
||||
|
||||
export function wrapTranscribeText(measure: (text: string) => number, text: string, maxWidth: number): string[] {
|
||||
const lines: string[] = [];
|
||||
for (const paragraph of text.split("\n")) {
|
||||
let line = "";
|
||||
for (const word of paragraph.split(/\s+/).filter(Boolean)) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && measure(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = "";
|
||||
}
|
||||
if (measure(word) <= maxWidth) {
|
||||
line = line ? `${line} ${word}` : word;
|
||||
continue;
|
||||
}
|
||||
// Long URLs and unspaced languages must not run off the display.
|
||||
for (const char of (line ? " " : "") + word) {
|
||||
if (line && measure(line + char) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = "";
|
||||
}
|
||||
line += char;
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
Reference in new issue
Block a user