forked from Blendman974/faceclaw
Add teleprompter app
This commit is contained in:
1 parent
b69fdc220a
commit
758cf7de3b
10 files changed
+986
-1
No files matched your search
@@ -14,6 +14,7 @@
|
|||||||
* Navigate app: Darken the map background color
|
* Navigate app: Darken the map background color
|
||||||
* EvenHub app: Add New, Search, and Updates tabs
|
* EvenHub app: Add New, Search, and Updates tabs
|
||||||
* Timer app: Fully rewritten, with new better UI
|
* Timer app: Fully rewritten, with new better UI
|
||||||
|
* Add a teleprompter app
|
||||||
|
|
||||||
## 0.6.4 - 2026-09-02
|
## 0.6.4 - 2026-09-02
|
||||||
* Context menus are centered and include window title
|
* Context menus are centered and include window title
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import filesApp from "./files";
|
|||||||
import musicApp from "./music";
|
import musicApp from "./music";
|
||||||
import nightscoutApp from "./nightscout";
|
import nightscoutApp from "./nightscout";
|
||||||
import transcribeApp from "./transcribe";
|
import transcribeApp from "./transcribe";
|
||||||
|
import teleprompterApp from "./teleprompter";
|
||||||
import microphonesApp from "./microphones";
|
import microphonesApp from "./microphones";
|
||||||
import notificationsApp from "./notifications";
|
import notificationsApp from "./notifications";
|
||||||
import calendarApp from "./calendar";
|
import calendarApp from "./calendar";
|
||||||
@@ -36,6 +37,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
|
|||||||
musicApp,
|
musicApp,
|
||||||
nightscoutApp,
|
nightscoutApp,
|
||||||
transcribeApp,
|
transcribeApp,
|
||||||
|
teleprompterApp,
|
||||||
microphonesApp,
|
microphonesApp,
|
||||||
notificationsApp,
|
notificationsApp,
|
||||||
calendarApp,
|
calendarApp,
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import { type AppDefinition } from "../app-definition";
|
||||||
|
import { createTeleprompterAppWindow, TELEPROMPTER_SURFACE_ID, TELEPROMPTER_WINDOW_ID } from "./teleprompter-app";
|
||||||
|
|
||||||
|
const teleprompterApp: AppDefinition = {
|
||||||
|
appId: "teleprompter",
|
||||||
|
title: "Teleprompter",
|
||||||
|
icon: "scroll-text",
|
||||||
|
launch: (ctx) =>
|
||||||
|
ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) =>
|
||||||
|
createTeleprompterAppWindow({
|
||||||
|
...options,
|
||||||
|
startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(),
|
||||||
|
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(),
|
||||||
|
appendLog: (message) => ctx.appendLog(message),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
export default teleprompterApp;
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||||
|
import { truncateText } from "../../graphics/textwrap";
|
||||||
|
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
|
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||||
|
import { type InputEvent } from "../../ui/gestures";
|
||||||
|
import { Layer, type LayerContext } from "../../ui/layers";
|
||||||
|
import { type MenuItem } from "../../ui/menu";
|
||||||
|
import { lineStep } from "../../ui/metrics";
|
||||||
|
import { clamp } from "../../util/numeric-util";
|
||||||
|
import { ScriptTracker, tokenizeScript, type TokenizedScript } from "./script-tracker";
|
||||||
|
|
||||||
|
const MARGIN_X = 18;
|
||||||
|
const BODY_TOP = 6;
|
||||||
|
/** Gap between the last body line and the footer. */
|
||||||
|
const FOOTER_GAP = 4;
|
||||||
|
/** Lines per ring scroll tick. */
|
||||||
|
const SCROLL_STEP_LINES = 2;
|
||||||
|
|
||||||
|
/** Brightness of words already spoken, still to come, and the next one. */
|
||||||
|
const PAST_VALUE = 90;
|
||||||
|
const UPCOMING_VALUE = 200;
|
||||||
|
const NEXT_VALUE = 255;
|
||||||
|
|
||||||
|
type LayoutSpan = { tokenIndex: number; x: number; width: number; text: string };
|
||||||
|
type LayoutLine = { spans: LayoutSpan[] };
|
||||||
|
|
||||||
|
export type ReaderHooks = {
|
||||||
|
requestRender: () => void;
|
||||||
|
/** Hold continuous mic capture while tracking is on. */
|
||||||
|
startCapture: () => void;
|
||||||
|
stopCapture: () => void;
|
||||||
|
/** The reader closed (or paused) at this word position. */
|
||||||
|
savePosition: (position: number) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The teleprompter's reading view. Lays the script out word by word in the
|
||||||
|
* medium UI font, follows the speaker through it from the live transcript
|
||||||
|
* (see ScriptTracker), auto-scrolls to keep the next word about a third of
|
||||||
|
* the way down, and highlights that word. Ring scrolls move the view by hand
|
||||||
|
* and re-anchor the tracker to what is now at the anchor row, so scrolling
|
||||||
|
* doubles as "I'm here now". Click pauses/resumes tracking; double-click
|
||||||
|
* goes back to the file list.
|
||||||
|
*/
|
||||||
|
export class TeleprompterReaderLayer implements Layer {
|
||||||
|
private readonly script: TokenizedScript;
|
||||||
|
private readonly tracker: ScriptTracker;
|
||||||
|
private lines: LayoutLine[] | null = null;
|
||||||
|
/** Line index of each word (parallel to script.words). */
|
||||||
|
private wordLines: number[] = [];
|
||||||
|
private layoutKey = "";
|
||||||
|
private firstLine = 0;
|
||||||
|
/** True after the tracker moved: the next paint re-centers on the position. */
|
||||||
|
private autoScrollPending = true;
|
||||||
|
private tracking = true;
|
||||||
|
private status = "";
|
||||||
|
private unsubscribeTranscript: (() => void) | null = null;
|
||||||
|
private unsubscribeStatus: (() => void) | null = null;
|
||||||
|
// Geometry from the last paint, for the input handlers.
|
||||||
|
private visibleRows = 10;
|
||||||
|
private anchorRow = 3;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
text: string,
|
||||||
|
private readonly title: string,
|
||||||
|
private readonly hooks: ReaderHooks,
|
||||||
|
initialPosition = 0,
|
||||||
|
) {
|
||||||
|
this.script = tokenizeScript(text);
|
||||||
|
this.tracker = new ScriptTracker(this.script.words);
|
||||||
|
this.tracker.anchor(initialPosition);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Start listening. Called once the layer is on its stack. */
|
||||||
|
attach(): void {
|
||||||
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||||
|
this.status = state.status;
|
||||||
|
this.hooks.requestRender();
|
||||||
|
});
|
||||||
|
this.startTracking();
|
||||||
|
}
|
||||||
|
|
||||||
|
onRemoved(): void {
|
||||||
|
this.hooks.savePosition(this.resumePosition());
|
||||||
|
this.stopTracking();
|
||||||
|
this.unsubscribeStatus?.();
|
||||||
|
this.unsubscribeStatus = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
get position(): number {
|
||||||
|
return this.tracker.position;
|
||||||
|
}
|
||||||
|
|
||||||
|
buildMenuItems(): MenuItem[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
label: this.tracking ? "Pause voice tracking" : "Resume voice tracking",
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
this.toggleTracking();
|
||||||
|
ctx.stack.pop();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Restart from the top",
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
this.tracker.anchor(0);
|
||||||
|
this.firstLine = 0;
|
||||||
|
this.autoScrollPending = true;
|
||||||
|
ctx.stack.pop();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
paint(ctx: LayerContext): GrayImage {
|
||||||
|
const font = getDefaultMediumFont();
|
||||||
|
const smallFont = getDefaultSmallFont();
|
||||||
|
const step = lineStep(font);
|
||||||
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
|
const image = new GrayImage(width, height, 0);
|
||||||
|
|
||||||
|
const footerY = height - smallFont.lineHeight - 4;
|
||||||
|
this.visibleRows = Math.max(1, Math.floor((footerY - FOOTER_GAP - BODY_TOP) / step));
|
||||||
|
this.anchorRow = Math.floor(this.visibleRows / 3);
|
||||||
|
const lines = this.layout(font, width - 2 * MARGIN_X);
|
||||||
|
|
||||||
|
if (this.autoScrollPending) {
|
||||||
|
this.autoScrollPending = false;
|
||||||
|
this.firstLine = this.clampFirstLine(this.lineOfPosition() - this.anchorRow);
|
||||||
|
} else {
|
||||||
|
this.firstLine = this.clampFirstLine(this.firstLine);
|
||||||
|
}
|
||||||
|
|
||||||
|
const position = this.tracker.position;
|
||||||
|
const nextToken = position < this.script.words.length ? this.script.wordTokens[position]! : this.script.tokens.length;
|
||||||
|
const lastLine = Math.min(lines.length, this.firstLine + this.visibleRows);
|
||||||
|
for (let lineIndex = this.firstLine; lineIndex < lastLine; lineIndex++) {
|
||||||
|
const y = BODY_TOP + (lineIndex - this.firstLine) * step;
|
||||||
|
for (const span of lines[lineIndex]!.spans) {
|
||||||
|
const isNext = span.tokenIndex === nextToken;
|
||||||
|
const value = isNext ? NEXT_VALUE : span.tokenIndex < nextToken ? PAST_VALUE : UPCOMING_VALUE;
|
||||||
|
image.drawText(font, MARGIN_X + span.x, y, span.text, value);
|
||||||
|
if (isNext) {
|
||||||
|
image.fillRect(MARGIN_X + span.x, y + font.lineHeight - 2, span.width, 2, NEXT_VALUE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.paintFooter(image, smallFont, width, footerY);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||||
|
switch (event.type) {
|
||||||
|
case "scroll-down":
|
||||||
|
this.scrollBy(SCROLL_STEP_LINES);
|
||||||
|
return;
|
||||||
|
case "scroll-up":
|
||||||
|
this.scrollBy(-SCROLL_STEP_LINES);
|
||||||
|
return;
|
||||||
|
case "click":
|
||||||
|
this.toggleTracking();
|
||||||
|
return;
|
||||||
|
case "double-click":
|
||||||
|
ctx.stack.pop();
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private paintFooter(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||||
|
const total = this.script.words.length;
|
||||||
|
const percent = total === 0 ? 0 : Math.round((100 * this.tracker.position) / total);
|
||||||
|
const right = `${this.tracking ? this.status || "Listening" : "Paused"} ${percent}%`;
|
||||||
|
const rightWidth = font.measureText(right);
|
||||||
|
image.drawText(font, width - MARGIN_X - rightWidth, y, right, 110);
|
||||||
|
const titleMax = width - 2 * MARGIN_X - rightWidth - 12;
|
||||||
|
if (titleMax > 20) {
|
||||||
|
image.drawText(font, MARGIN_X, y, truncateText(font, this.title, titleMax), 110);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hand-scroll by whole lines. The tracker is re-anchored to the first word
|
||||||
|
* at (or after) the anchor row, so the highlight follows the view and
|
||||||
|
* speech matching resumes from there.
|
||||||
|
*/
|
||||||
|
private scrollBy(deltaLines: number): void {
|
||||||
|
const lines = this.lines;
|
||||||
|
if (!lines) return;
|
||||||
|
this.autoScrollPending = false;
|
||||||
|
this.firstLine = this.clampFirstLine(this.firstLine + deltaLines);
|
||||||
|
const anchorLine = this.firstLine + this.anchorRow;
|
||||||
|
const word = this.firstWordAtOrAfterLine(anchorLine);
|
||||||
|
this.tracker.anchor(word);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clampFirstLine(value: number): number {
|
||||||
|
const lineCount = this.lines?.length ?? 0;
|
||||||
|
// Scroll far enough that the last line can reach the anchor row.
|
||||||
|
return clamp(value, 0, Math.max(0, lineCount - this.anchorRow - 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
private lineOfPosition(): number {
|
||||||
|
const position = this.tracker.position;
|
||||||
|
if (this.wordLines.length === 0) return 0;
|
||||||
|
if (position >= this.wordLines.length) return this.wordLines[this.wordLines.length - 1]!;
|
||||||
|
return this.wordLines[position]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
private firstWordAtOrAfterLine(line: number): number {
|
||||||
|
for (let word = 0; word < this.wordLines.length; word++) {
|
||||||
|
if (this.wordLines[word]! >= line) return word;
|
||||||
|
}
|
||||||
|
return this.wordLines.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Where to resume next time: a script read to the end restarts from the top. */
|
||||||
|
private resumePosition(): number {
|
||||||
|
return this.tracker.position >= this.script.words.length ? 0 : this.tracker.position;
|
||||||
|
}
|
||||||
|
|
||||||
|
private toggleTracking(): void {
|
||||||
|
if (this.tracking) {
|
||||||
|
this.stopTracking();
|
||||||
|
this.hooks.savePosition(this.resumePosition());
|
||||||
|
} else {
|
||||||
|
this.startTracking();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private startTracking(): void {
|
||||||
|
this.tracking = true;
|
||||||
|
if (!this.unsubscribeTranscript) {
|
||||||
|
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||||
|
}
|
||||||
|
this.hooks.startCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
private stopTracking(): void {
|
||||||
|
this.tracking = false;
|
||||||
|
this.unsubscribeTranscript?.();
|
||||||
|
this.unsubscribeTranscript = null;
|
||||||
|
this.hooks.stopCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||||
|
if (!this.tracking) return;
|
||||||
|
if (this.tracker.feed(event.text)) {
|
||||||
|
this.autoScrollPending = true;
|
||||||
|
this.hooks.requestRender();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Word-by-word greedy line layout; cached per font and width. */
|
||||||
|
private layout(font: UiFont, maxWidth: number): LayoutLine[] {
|
||||||
|
const key = `${font.fingerprintId}:${maxWidth}`;
|
||||||
|
if (this.lines && this.layoutKey === key) return this.lines;
|
||||||
|
const spaceWidth = font.measureText(" ");
|
||||||
|
const lines: LayoutLine[] = [];
|
||||||
|
const wordLines: number[] = [];
|
||||||
|
let current: LayoutLine = { spans: [] };
|
||||||
|
let currentWidth = 0;
|
||||||
|
let paragraph = -1;
|
||||||
|
const flush = () => {
|
||||||
|
lines.push(current);
|
||||||
|
current = { spans: [] };
|
||||||
|
currentWidth = 0;
|
||||||
|
};
|
||||||
|
for (let tokenIndex = 0; tokenIndex < this.script.tokens.length; tokenIndex++) {
|
||||||
|
const token = this.script.tokens[tokenIndex]!;
|
||||||
|
if (token.paragraph !== paragraph) {
|
||||||
|
// New paragraph: end the current line, and leave one empty line per
|
||||||
|
// blank source line (paragraph numbers count every newline).
|
||||||
|
if (paragraph >= 0) {
|
||||||
|
flush();
|
||||||
|
for (let blank = paragraph + 1; blank < token.paragraph; blank++) flush();
|
||||||
|
}
|
||||||
|
paragraph = token.paragraph;
|
||||||
|
}
|
||||||
|
const wordWidth = font.measureText(token.text);
|
||||||
|
const x = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
||||||
|
if (current.spans.length > 0 && x + wordWidth > maxWidth) {
|
||||||
|
flush();
|
||||||
|
}
|
||||||
|
const spanX = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
||||||
|
current.spans.push({ tokenIndex, x: spanX, width: wordWidth, text: token.text });
|
||||||
|
currentWidth = spanX + wordWidth;
|
||||||
|
if (token.wordIndex >= 0) {
|
||||||
|
wordLines[token.wordIndex] = lines.length;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (current.spans.length > 0 || lines.length === 0) flush();
|
||||||
|
this.lines = lines;
|
||||||
|
this.wordLines = wordLines;
|
||||||
|
this.layoutKey = key;
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Teleprompter's recently opened scripts, most recent first, with the
|
||||||
|
* word position the reader was at when it closed (so reopening resumes
|
||||||
|
* there). Stored as a JSON array in the settings store.
|
||||||
|
*/
|
||||||
|
export type RecentScript = {
|
||||||
|
path: string;
|
||||||
|
name: string;
|
||||||
|
/** Word index the reader was at when last closed; 0 = start. */
|
||||||
|
position: number;
|
||||||
|
openedMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const RECENTS_KEY = "teleprompter.recents";
|
||||||
|
const MAX_RECENTS = 12;
|
||||||
|
|
||||||
|
export function getRecentScripts(): RecentScript[] {
|
||||||
|
try {
|
||||||
|
const parsed: unknown = JSON.parse(getStringSetting(RECENTS_KEY, "[]"));
|
||||||
|
if (!Array.isArray(parsed)) return [];
|
||||||
|
return parsed
|
||||||
|
.filter((entry): entry is RecentScript => !!entry && typeof entry === "object" && typeof entry.path === "string")
|
||||||
|
.map((entry) => ({
|
||||||
|
path: entry.path,
|
||||||
|
name: typeof entry.name === "string" && entry.name ? entry.name : baseName(entry.path),
|
||||||
|
position: typeof entry.position === "number" && entry.position >= 0 ? entry.position : 0,
|
||||||
|
openedMs: typeof entry.openedMs === "number" ? entry.openedMs : 0,
|
||||||
|
}));
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveRecentScripts(recents: RecentScript[]): void {
|
||||||
|
setStringSetting(RECENTS_KEY, JSON.stringify(recents.slice(0, MAX_RECENTS)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Move (or add) a script to the top of the list, keeping its saved position. */
|
||||||
|
export function touchRecentScript(path: string, name: string): RecentScript {
|
||||||
|
const recents = getRecentScripts();
|
||||||
|
const existing = recents.find((entry) => entry.path === path);
|
||||||
|
const entry: RecentScript = {
|
||||||
|
path,
|
||||||
|
name,
|
||||||
|
position: existing?.position ?? 0,
|
||||||
|
openedMs: Date.now(),
|
||||||
|
};
|
||||||
|
saveRecentScripts([entry, ...recents.filter((other) => other.path !== path)]);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function saveRecentScriptPosition(path: string, position: number): void {
|
||||||
|
const recents = getRecentScripts();
|
||||||
|
const entry = recents.find((other) => other.path === path);
|
||||||
|
if (!entry) return;
|
||||||
|
entry.position = Math.max(0, position);
|
||||||
|
saveRecentScripts(recents);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function removeRecentScript(path: string): void {
|
||||||
|
saveRecentScripts(getRecentScripts().filter((entry) => entry.path !== path));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearRecentScripts(): void {
|
||||||
|
saveRecentScripts([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function baseName(path: string): string {
|
||||||
|
const slash = path.lastIndexOf("/");
|
||||||
|
return slash >= 0 ? path.slice(slash + 1) : path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The last directory segment of a file path, for a dim "where it lives" hint. */
|
||||||
|
export function parentDirName(path: string): string {
|
||||||
|
const slash = path.lastIndexOf("/");
|
||||||
|
if (slash <= 0) return "";
|
||||||
|
return baseName(path.slice(0, slash));
|
||||||
|
}
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
/**
|
||||||
|
* Follows a speaker through a script from live speech-to-text. Pure logic
|
||||||
|
* (no NativeScript imports) so it runs under plain node for the tests.
|
||||||
|
*
|
||||||
|
* The script is tokenized once into display tokens; the subset with a
|
||||||
|
* non-empty normalized key are the "words" the tracker aligns against. The
|
||||||
|
* tracker keeps `position`: the index (in word space) of the next word the
|
||||||
|
* speaker is expected to say. Each transcript update (replace semantics: the
|
||||||
|
* whole current utterance, partial or final) re-aligns the tail of the
|
||||||
|
* transcript against a window of script words around the current position,
|
||||||
|
* with a local alignment that tolerates misrecognized, skipped and ad-libbed
|
||||||
|
* words, and moves the position to just after the last aligned script word.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type ScriptToken = {
|
||||||
|
/** The token as written, for display. */
|
||||||
|
text: string;
|
||||||
|
/** Paragraph number (newline-separated), for layout. */
|
||||||
|
paragraph: number;
|
||||||
|
/** Lowercased alphanumeric key, or "" for punctuation-only tokens. */
|
||||||
|
key: string;
|
||||||
|
/** Index into the tracker's word list, or -1 when key is empty. */
|
||||||
|
wordIndex: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type TokenizedScript = {
|
||||||
|
tokens: ScriptToken[];
|
||||||
|
/** Normalized keys of the trackable tokens, in order. */
|
||||||
|
words: string[];
|
||||||
|
/** Token index of each word. */
|
||||||
|
wordTokens: number[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Lowercase, strip everything but letters/digits, fold common apostrophes. */
|
||||||
|
export function normalizeWord(raw: string): string {
|
||||||
|
return raw
|
||||||
|
.normalize("NFKD")
|
||||||
|
.replace(/[\u0300-\u036f]/g, "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tokenizeScript(text: string): TokenizedScript {
|
||||||
|
const tokens: ScriptToken[] = [];
|
||||||
|
const words: string[] = [];
|
||||||
|
const wordTokens: number[] = [];
|
||||||
|
const paragraphs = text.replace(/\r/g, "").split("\n");
|
||||||
|
for (let paragraph = 0; paragraph < paragraphs.length; paragraph++) {
|
||||||
|
for (const raw of paragraphs[paragraph]!.split(/\s+/)) {
|
||||||
|
if (!raw) continue;
|
||||||
|
const key = normalizeWord(raw);
|
||||||
|
const wordIndex = key ? words.length : -1;
|
||||||
|
if (key) {
|
||||||
|
words.push(key);
|
||||||
|
wordTokens.push(tokens.length);
|
||||||
|
}
|
||||||
|
tokens.push({ text: raw, paragraph, key, wordIndex });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { tokens, words, wordTokens };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Split a transcript into normalized keys, dropping empties. */
|
||||||
|
export function transcriptKeys(text: string): string[] {
|
||||||
|
return text
|
||||||
|
.split(/\s+/)
|
||||||
|
.map(normalizeWord)
|
||||||
|
.filter((key) => key.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function words match so often that on their own they say little about
|
||||||
|
// where the speaker is; they count for less than content words.
|
||||||
|
const STOP_WORDS = new Set([
|
||||||
|
"a", "an", "the", "and", "or", "but", "of", "to", "in", "on", "at", "for", "is", "it", "its",
|
||||||
|
"i", "we", "you", "he", "she", "they", "that", "this", "as", "be", "are", "was", "were",
|
||||||
|
"with", "by", "so", "if", "not", "no", "do", "have", "has", "had", "will", "can",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const MATCH_SCORE = 3;
|
||||||
|
const STOP_WORD_MATCH_SCORE = 1.5;
|
||||||
|
/** A fuzzy (edit-distance 1) or prefix match: real evidence, but weaker. */
|
||||||
|
const FUZZY_MATCH_SCORE = 2;
|
||||||
|
const MISMATCH_SCORE = -2;
|
||||||
|
const GAP_SCORE = -1.5;
|
||||||
|
/** The alignment must be worth at least this to move the position at all. */
|
||||||
|
const MIN_ALIGNMENT_SCORE = 4.5;
|
||||||
|
const MIN_MATCHED_WORDS = 2;
|
||||||
|
/** Per-word cost of ending an alignment away from the expected position (a tie-breaker). */
|
||||||
|
const FORWARD_DISTANCE_COST = 0.02;
|
||||||
|
const BACKWARD_DISTANCE_COST = 0.1;
|
||||||
|
|
||||||
|
export type TrackerOptions = {
|
||||||
|
/** Script words before the current position that stay in play (repeats, backtracking). */
|
||||||
|
back?: number;
|
||||||
|
/** Script words after the current position that stay in play (skipping ahead). */
|
||||||
|
ahead?: number;
|
||||||
|
/** Only the last this-many transcript words are aligned. */
|
||||||
|
tailWords?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_OPTIONS: Required<TrackerOptions> = { back: 30, ahead: 160, tailWords: 20 };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Edit distance capped at 1 (true when the strings are within one edit),
|
||||||
|
* which covers the typical STT slip: a dropped/extra letter or a substitution.
|
||||||
|
*/
|
||||||
|
export function withinOneEdit(a: string, b: string): boolean {
|
||||||
|
if (a === b) return true;
|
||||||
|
const lengthDelta = a.length - b.length;
|
||||||
|
if (lengthDelta > 1 || lengthDelta < -1) return false;
|
||||||
|
const [short, long] = a.length <= b.length ? [a, b] : [b, a];
|
||||||
|
let i = 0;
|
||||||
|
let j = 0;
|
||||||
|
let edits = 0;
|
||||||
|
while (i < short.length && j < long.length) {
|
||||||
|
if (short[i] === long[j]) {
|
||||||
|
i++;
|
||||||
|
j++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (++edits > 1) return false;
|
||||||
|
if (short.length === long.length) {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
return edits + (long.length - j) <= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Score of aligning transcript key `spoken` with script key `expected`. */
|
||||||
|
function matchScore(spoken: string, expected: string, isLastSpoken: boolean): number {
|
||||||
|
if (spoken === expected) {
|
||||||
|
return STOP_WORDS.has(expected) ? STOP_WORD_MATCH_SCORE : MATCH_SCORE;
|
||||||
|
}
|
||||||
|
// The final transcript word is often still being recognized: a prefix of
|
||||||
|
// the script word counts, once it is long enough not to be noise.
|
||||||
|
if (isLastSpoken && spoken.length >= 2 && expected.length > spoken.length && expected.startsWith(spoken)) {
|
||||||
|
return FUZZY_MATCH_SCORE;
|
||||||
|
}
|
||||||
|
if (spoken.length >= 5 && expected.length >= 5 && withinOneEdit(spoken, expected)) {
|
||||||
|
return FUZZY_MATCH_SCORE;
|
||||||
|
}
|
||||||
|
return MISMATCH_SCORE;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AlignmentResult = {
|
||||||
|
/** Index (in the script word list) of the word after the last aligned one; the new position. */
|
||||||
|
position: number;
|
||||||
|
score: number;
|
||||||
|
matchedWords: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Align the transcript tail against script words [windowStart, windowEnd).
|
||||||
|
* Local alignment (free start on both sides) that must consume the last
|
||||||
|
* transcript word and may end anywhere in the window. Returns null when the
|
||||||
|
* best alignment is too weak to trust.
|
||||||
|
*/
|
||||||
|
export function alignTail(
|
||||||
|
words: readonly string[],
|
||||||
|
spoken: readonly string[],
|
||||||
|
windowStart: number,
|
||||||
|
windowEnd: number,
|
||||||
|
expectedPosition: number,
|
||||||
|
): AlignmentResult | null {
|
||||||
|
const m = spoken.length;
|
||||||
|
const n = windowEnd - windowStart;
|
||||||
|
if (m === 0 || n <= 0) return null;
|
||||||
|
|
||||||
|
// H[i][j]: best score of an alignment ending with spoken[i-1] / window[j-1]
|
||||||
|
// (or a gap), clamped at 0 (free start). Matches counts how many spoken
|
||||||
|
// words matched along the path that produced H[i][j].
|
||||||
|
const width = n + 1;
|
||||||
|
const score = new Float64Array((m + 1) * width);
|
||||||
|
const matches = new Uint16Array((m + 1) * width);
|
||||||
|
for (let i = 1; i <= m; i++) {
|
||||||
|
const spokenWord = spoken[i - 1]!;
|
||||||
|
const isLast = i === m;
|
||||||
|
for (let j = 1; j <= n; j++) {
|
||||||
|
const here = i * width + j;
|
||||||
|
const s = matchScore(spokenWord, words[windowStart + j - 1]!, isLast);
|
||||||
|
const diagonal = score[here - width - 1]! + s;
|
||||||
|
const skipScript = score[here - 1]! + GAP_SCORE;
|
||||||
|
const skipSpoken = score[here - width]! + GAP_SCORE;
|
||||||
|
let best = 0;
|
||||||
|
let bestMatches = 0;
|
||||||
|
if (diagonal > best) {
|
||||||
|
best = diagonal;
|
||||||
|
bestMatches = matches[here - width - 1]! + (s > 0 ? 1 : 0);
|
||||||
|
}
|
||||||
|
if (skipScript > best) {
|
||||||
|
best = skipScript;
|
||||||
|
bestMatches = matches[here - 1]!;
|
||||||
|
}
|
||||||
|
if (skipSpoken > best) {
|
||||||
|
best = skipSpoken;
|
||||||
|
bestMatches = matches[here - width]!;
|
||||||
|
}
|
||||||
|
score[here] = best;
|
||||||
|
matches[here] = bestMatches;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let bestColumn = -1;
|
||||||
|
let bestAdjusted = -Infinity;
|
||||||
|
for (let j = 1; j <= n; j++) {
|
||||||
|
const raw = score[m * width + j]!;
|
||||||
|
if (raw <= 0) continue;
|
||||||
|
const position = windowStart + j;
|
||||||
|
const distance = position - expectedPosition;
|
||||||
|
const adjusted = raw - (distance >= 0 ? distance * FORWARD_DISTANCE_COST : -distance * BACKWARD_DISTANCE_COST);
|
||||||
|
if (adjusted > bestAdjusted) {
|
||||||
|
bestAdjusted = adjusted;
|
||||||
|
bestColumn = j;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bestColumn < 0) return null;
|
||||||
|
const rawScore = score[m * width + bestColumn]!;
|
||||||
|
const matchedWords = matches[m * width + bestColumn]!;
|
||||||
|
if (rawScore < MIN_ALIGNMENT_SCORE || matchedWords < MIN_MATCHED_WORDS) return null;
|
||||||
|
return { position: windowStart + bestColumn, score: rawScore, matchedWords };
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ScriptTracker {
|
||||||
|
private readonly options: Required<TrackerOptions>;
|
||||||
|
/** Index of the next word the speaker is expected to say (0..words.length). */
|
||||||
|
position = 0;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly words: readonly string[],
|
||||||
|
options: TrackerOptions = {},
|
||||||
|
) {
|
||||||
|
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||||
|
}
|
||||||
|
|
||||||
|
get wordCount(): number {
|
||||||
|
return this.words.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Jump the expected position (manual scroll, restart). */
|
||||||
|
anchor(position: number): void {
|
||||||
|
this.position = Math.max(0, Math.min(this.words.length, position));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply a transcript update (the whole current utterance, partial or
|
||||||
|
* final). Returns true when the position moved.
|
||||||
|
*/
|
||||||
|
feed(text: string): boolean {
|
||||||
|
const keys = transcriptKeys(text);
|
||||||
|
if (keys.length === 0) return false;
|
||||||
|
const tail = keys.length > this.options.tailWords ? keys.slice(-this.options.tailWords) : keys;
|
||||||
|
const windowStart = Math.max(0, this.position - this.options.back);
|
||||||
|
const windowEnd = Math.min(this.words.length, this.position + this.options.ahead);
|
||||||
|
const result = alignTail(this.words, tail, windowStart, windowEnd, this.position);
|
||||||
|
if (!result || result.position === this.position) return false;
|
||||||
|
this.position = result.position;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
|
import { truncateText } from "../../graphics/textwrap";
|
||||||
|
import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||||
|
import { LIST_ROW_TEXT_INSET } from "../../ui/metrics";
|
||||||
|
import { MenuLayer, type MenuItem } from "../../ui/menu";
|
||||||
|
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||||
|
import { appViewportSize } from "../../ui/shell/geometry";
|
||||||
|
import {
|
||||||
|
createInProcessWindow,
|
||||||
|
YieldAtRootLayer,
|
||||||
|
type InProcessAppOptions,
|
||||||
|
type InProcessWindow,
|
||||||
|
} from "../../ui/shell/in-process-window";
|
||||||
|
import { FileBrowserLayer } from "../files/file-browser";
|
||||||
|
import { TeleprompterReaderLayer } from "./reader";
|
||||||
|
import {
|
||||||
|
baseName,
|
||||||
|
clearRecentScripts,
|
||||||
|
getRecentScripts,
|
||||||
|
parentDirName,
|
||||||
|
removeRecentScript,
|
||||||
|
saveRecentScriptPosition,
|
||||||
|
touchRecentScript,
|
||||||
|
} from "./recent-files";
|
||||||
|
|
||||||
|
export const TELEPROMPTER_WINDOW_ID = "teleprompter";
|
||||||
|
export const TELEPROMPTER_SURFACE_ID = "window:teleprompter";
|
||||||
|
|
||||||
|
const SCRIPT_FILE = /\.(txt|md|markdown|text|log)$/i;
|
||||||
|
|
||||||
|
const HOME_LAYOUT = {
|
||||||
|
x: 8,
|
||||||
|
y: 8,
|
||||||
|
width: 272,
|
||||||
|
showBorder: false,
|
||||||
|
minHeight: 0,
|
||||||
|
maxHeight: appViewportSize("min").height - 16,
|
||||||
|
opaque: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
export type TeleprompterAppOptions = InProcessAppOptions & {
|
||||||
|
/** Hold continuous mic capture while a script is being tracked. */
|
||||||
|
startContinuousVoiceCapture: () => void;
|
||||||
|
stopContinuousVoiceCapture: () => void;
|
||||||
|
appendLog: (message: string) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Teleprompter app window. The home page is a Browse entry plus the
|
||||||
|
* recently opened scripts; Browse pushes the Files app's browser to pick a
|
||||||
|
* text file, and either path pushes the reader (see TeleprompterReaderLayer)
|
||||||
|
* over the home page.
|
||||||
|
*/
|
||||||
|
export function createTeleprompterAppWindow(options: TeleprompterAppOptions): InProcessWindow {
|
||||||
|
// The home menu's items are rebuilt in place whenever the recents change
|
||||||
|
// (MenuLayer reads the array each paint).
|
||||||
|
const homeItems: MenuItem[] = [];
|
||||||
|
const home = new MenuLayer("Teleprompter", homeItems, HOME_LAYOUT);
|
||||||
|
let created: InProcessWindow | null = null;
|
||||||
|
let notice = "";
|
||||||
|
|
||||||
|
const requestRender = () => created?.requestRender();
|
||||||
|
|
||||||
|
/** The layer currently on top of the window's stack (the home menu at base). */
|
||||||
|
const topLayer = (): Layer => {
|
||||||
|
let top: Layer = home;
|
||||||
|
created?.stack.topMatches((layer) => {
|
||||||
|
top = layer;
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return top;
|
||||||
|
};
|
||||||
|
|
||||||
|
const openScript = (ctx: LayerContext, path: string, name: string) => {
|
||||||
|
const text = readTextFile(path);
|
||||||
|
if (text === null) {
|
||||||
|
removeRecentScript(path);
|
||||||
|
notice = `Could not read ${name}`;
|
||||||
|
options.appendLog(`teleprompter: could not read ${path}`);
|
||||||
|
refreshHome();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const recent = touchRecentScript(path, name);
|
||||||
|
notice = "";
|
||||||
|
refreshHome();
|
||||||
|
const layer = new TeleprompterReaderLayer(
|
||||||
|
text,
|
||||||
|
name,
|
||||||
|
{
|
||||||
|
requestRender,
|
||||||
|
startCapture: () => options.startContinuousVoiceCapture(),
|
||||||
|
stopCapture: () => options.stopContinuousVoiceCapture(),
|
||||||
|
savePosition: (position) => saveRecentScriptPosition(path, position),
|
||||||
|
},
|
||||||
|
recent.position,
|
||||||
|
);
|
||||||
|
ctx.stack.push(layer);
|
||||||
|
layer.attach();
|
||||||
|
};
|
||||||
|
|
||||||
|
const openBrowser = (ctx: LayerContext) => {
|
||||||
|
const stack = ctx.stack;
|
||||||
|
const picker = new FileBrowserLayer({
|
||||||
|
isSupportedFile: (name) => SCRIPT_FILE.test(name),
|
||||||
|
onLeave: () => stack.pop(),
|
||||||
|
onFilePicked: (entry: DirectoryEntry, pickCtx) => {
|
||||||
|
if (!SCRIPT_FILE.test(entry.name)) {
|
||||||
|
notice = "Pick a text file (.txt, .md)";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pickCtx.stack.pop();
|
||||||
|
openScript(pickCtx, entry.path, entry.name);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
stack.push(picker);
|
||||||
|
};
|
||||||
|
|
||||||
|
function refreshHome(): void {
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
const items: MenuItem[] = [
|
||||||
|
{
|
||||||
|
label: "Browse for a script...",
|
||||||
|
onSelect: (ctx) => openBrowser(ctx),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
for (const recent of getRecentScripts()) {
|
||||||
|
const folder = parentDirName(recent.path);
|
||||||
|
items.push({
|
||||||
|
label: recent.name || baseName(recent.path),
|
||||||
|
onSelect: (ctx) => openScript(ctx, recent.path, recent.name),
|
||||||
|
render: ({ image, x, y, width, selected, text }) => {
|
||||||
|
const value = selected ? 255 : 200;
|
||||||
|
const folderWidth = folder ? font.measureText(folder) : 0;
|
||||||
|
const nameMax = folder ? width - folderWidth - 12 : width;
|
||||||
|
image.drawText(font, x, y + LIST_ROW_TEXT_INSET, truncateText(font, text, Math.max(40, nameMax)), value);
|
||||||
|
if (folder && nameMax >= 40) {
|
||||||
|
image.drawText(font, x + width - folderWidth, y + LIST_ROW_TEXT_INSET, folder, 90);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (notice) {
|
||||||
|
items.push({ label: notice, disabled: true, onSelect: () => {} });
|
||||||
|
}
|
||||||
|
homeItems.length = 0;
|
||||||
|
homeItems.push(...items);
|
||||||
|
// Land on the most recent script (one click resumes it), else on Browse.
|
||||||
|
home.selectItem(getRecentScripts().length > 0 ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
refreshHome();
|
||||||
|
|
||||||
|
created = createInProcessWindow({
|
||||||
|
appId: "teleprompter",
|
||||||
|
windowId: TELEPROMPTER_WINDOW_ID,
|
||||||
|
title: "Teleprompter",
|
||||||
|
iconLetter: "Tp",
|
||||||
|
icon: "scroll-text",
|
||||||
|
closeable: true,
|
||||||
|
actions: options.actions,
|
||||||
|
menuItems: () => {
|
||||||
|
const top = topLayer();
|
||||||
|
if (top instanceof TeleprompterReaderLayer) return top.buildMenuItems();
|
||||||
|
if (top instanceof FileBrowserLayer) return top.buildMenuItems();
|
||||||
|
if (top !== home) return [];
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
label: "Clear recent scripts",
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
clearRecentScripts();
|
||||||
|
refreshHome();
|
||||||
|
ctx.stack.pop();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
},
|
||||||
|
onFocus: (lastInput) => {
|
||||||
|
const top = topLayer();
|
||||||
|
if (top instanceof FileBrowserLayer) top.onFocus(lastInput);
|
||||||
|
},
|
||||||
|
baseLayer: new YieldAtRootLayer(home),
|
||||||
|
submitFrame: options.submitFrame,
|
||||||
|
setSurfaceVisible: options.setSurfaceVisible,
|
||||||
|
removeSurface: options.removeSurface,
|
||||||
|
onClosed: options.onClosed,
|
||||||
|
});
|
||||||
|
return created;
|
||||||
|
}
|
||||||
@@ -36,6 +36,8 @@ export const ICON_SVGS = {
|
|||||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M12 18v4"/><path d="M2 14.499a5.5 5.5 0 0 0 9.591 3.675.6.6 0 0 1 .818.001A5.5 5.5 0 0 0 22 14.5c0-2.29-1.5-4-3-5.5l-5.492-5.312a2 2 0 0 0-3-.02L5 8.999c-1.5 1.5-3 3.2-3 5.5"/></svg>',
|
'<svg viewBox="0 0 24 24" fill="none"><path d="M12 18v4"/><path d="M2 14.499a5.5 5.5 0 0 0 9.591 3.675.6.6 0 0 1 .818.001A5.5 5.5 0 0 0 22 14.5c0-2.29-1.5-4-3-5.5l-5.492-5.312a2 2 0 0 0-3-.02L5 8.999c-1.5 1.5-3 3.2-3 5.5"/></svg>',
|
||||||
terminal:
|
terminal:
|
||||||
'<svg viewBox="0 0 24 24" fill="none"><path d="m7 11 2-2-2-2"/><path d="M11 13h4"/><rect width="18" height="18" x="3" y="3" rx="2" ry="2"/></svg>',
|
'<svg viewBox="0 0 24 24" fill="none"><path d="m7 11 2-2-2-2"/><path d="M11 13h4"/><rect width="18" height="18" x="3" y="3" rx="2" ry="2"/></svg>',
|
||||||
|
"scroll-text":
|
||||||
|
'<svg viewBox="0 0 24 24" fill="none"><path d="M15 12h-5"/><path d="M15 8h-5"/><path d="M19 17V5a2 2 0 0 0-2-2H4"/><path d="M8 21h12a2 2 0 0 0 2-2v-1a1 1 0 0 0-1-1H11a1 1 0 0 0-1 1v1a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v2a1 1 0 0 0 1 1h3"/></svg>',
|
||||||
"file-text":
|
"file-text":
|
||||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M6 22a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h8a2.4 2.4 0 0 1 1.704.706l3.588 3.588A2.4 2.4 0 0 1 20 8v12a2 2 0 0 1-2 2z"/><path d="M14 2v5a1 1 0 0 0 1 1h5"/><path d="M10 9H8"/><path d="M16 13H8"/><path d="M16 17H8"/></svg>',
|
'<svg viewBox="0 0 24 24" fill="none"><path d="M6 22a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h8a2.4 2.4 0 0 1 1.704.706l3.588 3.588A2.4 2.4 0 0 1 20 8v12a2 2 0 0 1-2 2z"/><path d="M14 2v5a1 1 0 0 0 1 1h5"/><path d="M10 9H8"/><path d="M16 13H8"/><path d="M16 17H8"/></svg>',
|
||||||
file:
|
file:
|
||||||
|
|||||||
@@ -0,0 +1,132 @@
|
|||||||
|
// Pins the Teleprompter's speech-to-script tracker: tokenization, the fuzzy
|
||||||
|
// word matcher, and the position tracking under realistic partial-transcript
|
||||||
|
// streams (replace semantics), including misrecognitions, ad-libs, skipped
|
||||||
|
// sentences and repeated phrases.
|
||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
|
||||||
|
const tracker = require("../.test-build/app/apps/teleprompter/script-tracker.js");
|
||||||
|
|
||||||
|
const SCRIPT = `Good evening, everyone. Thank you for coming tonight.
|
||||||
|
|
||||||
|
Four score and seven years ago our fathers brought forth on this continent
|
||||||
|
a new nation, conceived in liberty, and dedicated to the proposition that
|
||||||
|
all men are created equal.
|
||||||
|
|
||||||
|
Now we are engaged in a great civil war, testing whether that nation, or any
|
||||||
|
nation so conceived and so dedicated, can long endure. We are met on a great
|
||||||
|
battle-field of that war.
|
||||||
|
|
||||||
|
We have come to dedicate a portion of that field, as a final resting place
|
||||||
|
for those who here gave their lives that that nation might live.`;
|
||||||
|
|
||||||
|
/** Word index of the first occurrence of a normalized key, from `from`. */
|
||||||
|
function wordAt(script, key, from = 0) {
|
||||||
|
const index = script.words.indexOf(key, from);
|
||||||
|
assert.notEqual(index, -1, `script has ${key}`);
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Feed an utterance the way a streaming recognizer delivers it: a growing
|
||||||
|
* partial after every word, then the final. Returns the tracker position.
|
||||||
|
*/
|
||||||
|
function speak(t, utterance) {
|
||||||
|
const words = utterance.split(/\s+/);
|
||||||
|
for (let i = 1; i <= words.length; i++) {
|
||||||
|
t.feed(words.slice(0, i).join(" "));
|
||||||
|
}
|
||||||
|
t.feed(utterance);
|
||||||
|
return t.position;
|
||||||
|
}
|
||||||
|
|
||||||
|
test("tokenizeScript separates display tokens from trackable words", () => {
|
||||||
|
const script = tracker.tokenizeScript("Hello, world — it's 2026.\nNext line");
|
||||||
|
assert.deepEqual(
|
||||||
|
script.tokens.map((token) => token.text),
|
||||||
|
["Hello,", "world", "—", "it's", "2026.", "Next", "line"],
|
||||||
|
);
|
||||||
|
assert.deepEqual(script.words, ["hello", "world", "its", "2026", "next", "line"]);
|
||||||
|
assert.equal(script.tokens[2].wordIndex, -1);
|
||||||
|
assert.equal(script.tokens[5].paragraph, 1);
|
||||||
|
assert.deepEqual(script.wordTokens, [0, 1, 3, 4, 5, 6]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("withinOneEdit accepts single slips and rejects more", () => {
|
||||||
|
assert.ok(tracker.withinOneEdit("conceived", "conceived"));
|
||||||
|
assert.ok(tracker.withinOneEdit("conceived", "concieved") === false, "transposition is two edits");
|
||||||
|
assert.ok(tracker.withinOneEdit("liberty", "libery"));
|
||||||
|
assert.ok(tracker.withinOneEdit("liberty", "libertyy"));
|
||||||
|
assert.ok(tracker.withinOneEdit("liberty", "libarty"));
|
||||||
|
assert.ok(!tracker.withinOneEdit("liberty", "library"));
|
||||||
|
assert.ok(tracker.withinOneEdit("nation", "notion"), "one substitution");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("follows a clean read-through, word by word", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
assert.equal(t.position, 0);
|
||||||
|
// A lone first word is not enough evidence to move.
|
||||||
|
t.feed("Good");
|
||||||
|
assert.equal(t.position, 0);
|
||||||
|
t.feed("Good evening");
|
||||||
|
assert.equal(t.position, 2);
|
||||||
|
speak(t, "Good evening everyone thank you for coming tonight");
|
||||||
|
assert.equal(t.position, wordAt(script, "four"));
|
||||||
|
speak(t, "Four score and seven years ago our fathers brought forth on this continent");
|
||||||
|
assert.equal(t.position, wordAt(script, "a", wordAt(script, "continent")));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("tolerates misrecognized and ad-libbed words", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
t.anchor(wordAt(script, "four"));
|
||||||
|
// "score" heard as "scored", "fathers" as "father's", plus an inserted "um".
|
||||||
|
speak(t, "for scored and seven years ago um our father's brought forth");
|
||||||
|
assert.equal(t.position, wordAt(script, "on"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a partial last word counts as a prefix of the next script word", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
t.anchor(wordAt(script, "four"));
|
||||||
|
t.feed("four score and seven years ago our fath");
|
||||||
|
assert.equal(t.position, wordAt(script, "brought"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("catches up when the speaker skips a sentence", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
t.anchor(wordAt(script, "four"));
|
||||||
|
speak(t, "Now we are engaged in a great civil war");
|
||||||
|
assert.equal(t.position, wordAt(script, "testing"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("prefers the occurrence nearest the current position for repeated phrases", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
// "that nation" appears in paragraphs 3 and 4; anchored in paragraph 4 it
|
||||||
|
// should pick the later one.
|
||||||
|
const laterThat = wordAt(script, "gave");
|
||||||
|
t.anchor(laterThat);
|
||||||
|
speak(t, "their lives that that nation might live");
|
||||||
|
assert.equal(t.position, script.words.length);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not move on transcripts that match nothing nearby", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words);
|
||||||
|
t.anchor(wordAt(script, "four"));
|
||||||
|
assert.equal(speak(t, "hang on let me find my glasses"), wordAt(script, "four"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("manual anchoring re-bases the search window", () => {
|
||||||
|
const script = tracker.tokenizeScript(SCRIPT);
|
||||||
|
const t = new tracker.ScriptTracker(script.words, { ahead: 10, back: 5 });
|
||||||
|
// Far outside the window from position 0.
|
||||||
|
speak(t, "final resting place for those who here gave");
|
||||||
|
assert.equal(t.position, 0);
|
||||||
|
t.anchor(wordAt(script, "final"));
|
||||||
|
speak(t, "final resting place for those who here gave");
|
||||||
|
assert.equal(t.position, wordAt(script, "their"));
|
||||||
|
});
|
||||||
+2
-1
@@ -25,6 +25,7 @@
|
|||||||
"../app/apps/microphones/radar-dots.ts",
|
"../app/apps/microphones/radar-dots.ts",
|
||||||
"../app/apps/microphones/name-guess.ts",
|
"../app/apps/microphones/name-guess.ts",
|
||||||
"../app/apps/evenhub/containers.ts",
|
"../app/apps/evenhub/containers.ts",
|
||||||
"../app/apps/timer/timer-model.ts"
|
"../app/apps/timer/timer-model.ts",
|
||||||
|
"../app/apps/teleprompter/script-tracker.ts"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user