Add teleprompter app
This commit is contained in:
1 parent
b69fdc220a
commit
758cf7de3b
10 files changed
+986
-1
No files matched your search
@@ -14,6 +14,7 @@
|
||||
* Navigate app: Darken the map background color
|
||||
* EvenHub app: Add New, Search, and Updates tabs
|
||||
* Timer app: Fully rewritten, with new better UI
|
||||
* Add a teleprompter app
|
||||
|
||||
## 0.6.4 - 2026-09-02
|
||||
* Context menus are centered and include window title
|
||||
|
||||
@@ -7,6 +7,7 @@ import filesApp from "./files";
|
||||
import musicApp from "./music";
|
||||
import nightscoutApp from "./nightscout";
|
||||
import transcribeApp from "./transcribe";
|
||||
import teleprompterApp from "./teleprompter";
|
||||
import microphonesApp from "./microphones";
|
||||
import notificationsApp from "./notifications";
|
||||
import calendarApp from "./calendar";
|
||||
@@ -36,6 +37,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
|
||||
musicApp,
|
||||
nightscoutApp,
|
||||
transcribeApp,
|
||||
teleprompterApp,
|
||||
microphonesApp,
|
||||
notificationsApp,
|
||||
calendarApp,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTeleprompterAppWindow, TELEPROMPTER_SURFACE_ID, TELEPROMPTER_WINDOW_ID } from "./teleprompter-app";
|
||||
|
||||
const teleprompterApp: AppDefinition = {
|
||||
appId: "teleprompter",
|
||||
title: "Teleprompter",
|
||||
icon: "scroll-text",
|
||||
launch: (ctx) =>
|
||||
ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) =>
|
||||
createTeleprompterAppWindow({
|
||||
...options,
|
||||
startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(),
|
||||
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(),
|
||||
appendLog: (message) => ctx.appendLog(message),
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
export default teleprompterApp;
|
||||
@@ -0,0 +1,300 @@
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { truncateText } from "../../graphics/textwrap";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type MenuItem } from "../../ui/menu";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import { ScriptTracker, tokenizeScript, type TokenizedScript } from "./script-tracker";
|
||||
|
||||
const MARGIN_X = 18;
|
||||
const BODY_TOP = 6;
|
||||
/** Gap between the last body line and the footer. */
|
||||
const FOOTER_GAP = 4;
|
||||
/** Lines per ring scroll tick. */
|
||||
const SCROLL_STEP_LINES = 2;
|
||||
|
||||
/** Brightness of words already spoken, still to come, and the next one. */
|
||||
const PAST_VALUE = 90;
|
||||
const UPCOMING_VALUE = 200;
|
||||
const NEXT_VALUE = 255;
|
||||
|
||||
type LayoutSpan = { tokenIndex: number; x: number; width: number; text: string };
|
||||
type LayoutLine = { spans: LayoutSpan[] };
|
||||
|
||||
export type ReaderHooks = {
|
||||
requestRender: () => void;
|
||||
/** Hold continuous mic capture while tracking is on. */
|
||||
startCapture: () => void;
|
||||
stopCapture: () => void;
|
||||
/** The reader closed (or paused) at this word position. */
|
||||
savePosition: (position: number) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The teleprompter's reading view. Lays the script out word by word in the
|
||||
* medium UI font, follows the speaker through it from the live transcript
|
||||
* (see ScriptTracker), auto-scrolls to keep the next word about a third of
|
||||
* the way down, and highlights that word. Ring scrolls move the view by hand
|
||||
* and re-anchor the tracker to what is now at the anchor row, so scrolling
|
||||
* doubles as "I'm here now". Click pauses/resumes tracking; double-click
|
||||
* goes back to the file list.
|
||||
*/
|
||||
export class TeleprompterReaderLayer implements Layer {
|
||||
private readonly script: TokenizedScript;
|
||||
private readonly tracker: ScriptTracker;
|
||||
private lines: LayoutLine[] | null = null;
|
||||
/** Line index of each word (parallel to script.words). */
|
||||
private wordLines: number[] = [];
|
||||
private layoutKey = "";
|
||||
private firstLine = 0;
|
||||
/** True after the tracker moved: the next paint re-centers on the position. */
|
||||
private autoScrollPending = true;
|
||||
private tracking = true;
|
||||
private status = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
// Geometry from the last paint, for the input handlers.
|
||||
private visibleRows = 10;
|
||||
private anchorRow = 3;
|
||||
|
||||
constructor(
|
||||
text: string,
|
||||
private readonly title: string,
|
||||
private readonly hooks: ReaderHooks,
|
||||
initialPosition = 0,
|
||||
) {
|
||||
this.script = tokenizeScript(text);
|
||||
this.tracker = new ScriptTracker(this.script.words);
|
||||
this.tracker.anchor(initialPosition);
|
||||
}
|
||||
|
||||
/** Start listening. Called once the layer is on its stack. */
|
||||
attach(): void {
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.hooks.requestRender();
|
||||
});
|
||||
this.startTracking();
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.hooks.savePosition(this.resumePosition());
|
||||
this.stopTracking();
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
}
|
||||
|
||||
get position(): number {
|
||||
return this.tracker.position;
|
||||
}
|
||||
|
||||
buildMenuItems(): MenuItem[] {
|
||||
return [
|
||||
{
|
||||
label: this.tracking ? "Pause voice tracking" : "Resume voice tracking",
|
||||
onSelect: (ctx) => {
|
||||
this.toggleTracking();
|
||||
ctx.stack.pop();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Restart from the top",
|
||||
onSelect: (ctx) => {
|
||||
this.tracker.anchor(0);
|
||||
this.firstLine = 0;
|
||||
this.autoScrollPending = true;
|
||||
ctx.stack.pop();
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultMediumFont();
|
||||
const smallFont = getDefaultSmallFont();
|
||||
const step = lineStep(font);
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
|
||||
const footerY = height - smallFont.lineHeight - 4;
|
||||
this.visibleRows = Math.max(1, Math.floor((footerY - FOOTER_GAP - BODY_TOP) / step));
|
||||
this.anchorRow = Math.floor(this.visibleRows / 3);
|
||||
const lines = this.layout(font, width - 2 * MARGIN_X);
|
||||
|
||||
if (this.autoScrollPending) {
|
||||
this.autoScrollPending = false;
|
||||
this.firstLine = this.clampFirstLine(this.lineOfPosition() - this.anchorRow);
|
||||
} else {
|
||||
this.firstLine = this.clampFirstLine(this.firstLine);
|
||||
}
|
||||
|
||||
const position = this.tracker.position;
|
||||
const nextToken = position < this.script.words.length ? this.script.wordTokens[position]! : this.script.tokens.length;
|
||||
const lastLine = Math.min(lines.length, this.firstLine + this.visibleRows);
|
||||
for (let lineIndex = this.firstLine; lineIndex < lastLine; lineIndex++) {
|
||||
const y = BODY_TOP + (lineIndex - this.firstLine) * step;
|
||||
for (const span of lines[lineIndex]!.spans) {
|
||||
const isNext = span.tokenIndex === nextToken;
|
||||
const value = isNext ? NEXT_VALUE : span.tokenIndex < nextToken ? PAST_VALUE : UPCOMING_VALUE;
|
||||
image.drawText(font, MARGIN_X + span.x, y, span.text, value);
|
||||
if (isNext) {
|
||||
image.fillRect(MARGIN_X + span.x, y + font.lineHeight - 2, span.width, 2, NEXT_VALUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.paintFooter(image, smallFont, width, footerY);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-down":
|
||||
this.scrollBy(SCROLL_STEP_LINES);
|
||||
return;
|
||||
case "scroll-up":
|
||||
this.scrollBy(-SCROLL_STEP_LINES);
|
||||
return;
|
||||
case "click":
|
||||
this.toggleTracking();
|
||||
return;
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private paintFooter(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||
const total = this.script.words.length;
|
||||
const percent = total === 0 ? 0 : Math.round((100 * this.tracker.position) / total);
|
||||
const right = `${this.tracking ? this.status || "Listening" : "Paused"} ${percent}%`;
|
||||
const rightWidth = font.measureText(right);
|
||||
image.drawText(font, width - MARGIN_X - rightWidth, y, right, 110);
|
||||
const titleMax = width - 2 * MARGIN_X - rightWidth - 12;
|
||||
if (titleMax > 20) {
|
||||
image.drawText(font, MARGIN_X, y, truncateText(font, this.title, titleMax), 110);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand-scroll by whole lines. The tracker is re-anchored to the first word
|
||||
* at (or after) the anchor row, so the highlight follows the view and
|
||||
* speech matching resumes from there.
|
||||
*/
|
||||
private scrollBy(deltaLines: number): void {
|
||||
const lines = this.lines;
|
||||
if (!lines) return;
|
||||
this.autoScrollPending = false;
|
||||
this.firstLine = this.clampFirstLine(this.firstLine + deltaLines);
|
||||
const anchorLine = this.firstLine + this.anchorRow;
|
||||
const word = this.firstWordAtOrAfterLine(anchorLine);
|
||||
this.tracker.anchor(word);
|
||||
}
|
||||
|
||||
private clampFirstLine(value: number): number {
|
||||
const lineCount = this.lines?.length ?? 0;
|
||||
// Scroll far enough that the last line can reach the anchor row.
|
||||
return clamp(value, 0, Math.max(0, lineCount - this.anchorRow - 1));
|
||||
}
|
||||
|
||||
private lineOfPosition(): number {
|
||||
const position = this.tracker.position;
|
||||
if (this.wordLines.length === 0) return 0;
|
||||
if (position >= this.wordLines.length) return this.wordLines[this.wordLines.length - 1]!;
|
||||
return this.wordLines[position]!;
|
||||
}
|
||||
|
||||
private firstWordAtOrAfterLine(line: number): number {
|
||||
for (let word = 0; word < this.wordLines.length; word++) {
|
||||
if (this.wordLines[word]! >= line) return word;
|
||||
}
|
||||
return this.wordLines.length;
|
||||
}
|
||||
|
||||
/** Where to resume next time: a script read to the end restarts from the top. */
|
||||
private resumePosition(): number {
|
||||
return this.tracker.position >= this.script.words.length ? 0 : this.tracker.position;
|
||||
}
|
||||
|
||||
private toggleTracking(): void {
|
||||
if (this.tracking) {
|
||||
this.stopTracking();
|
||||
this.hooks.savePosition(this.resumePosition());
|
||||
} else {
|
||||
this.startTracking();
|
||||
}
|
||||
}
|
||||
|
||||
private startTracking(): void {
|
||||
this.tracking = true;
|
||||
if (!this.unsubscribeTranscript) {
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
}
|
||||
this.hooks.startCapture();
|
||||
}
|
||||
|
||||
private stopTracking(): void {
|
||||
this.tracking = false;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.hooks.stopCapture();
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (!this.tracking) return;
|
||||
if (this.tracker.feed(event.text)) {
|
||||
this.autoScrollPending = true;
|
||||
this.hooks.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
/** Word-by-word greedy line layout; cached per font and width. */
|
||||
private layout(font: UiFont, maxWidth: number): LayoutLine[] {
|
||||
const key = `${font.fingerprintId}:${maxWidth}`;
|
||||
if (this.lines && this.layoutKey === key) return this.lines;
|
||||
const spaceWidth = font.measureText(" ");
|
||||
const lines: LayoutLine[] = [];
|
||||
const wordLines: number[] = [];
|
||||
let current: LayoutLine = { spans: [] };
|
||||
let currentWidth = 0;
|
||||
let paragraph = -1;
|
||||
const flush = () => {
|
||||
lines.push(current);
|
||||
current = { spans: [] };
|
||||
currentWidth = 0;
|
||||
};
|
||||
for (let tokenIndex = 0; tokenIndex < this.script.tokens.length; tokenIndex++) {
|
||||
const token = this.script.tokens[tokenIndex]!;
|
||||
if (token.paragraph !== paragraph) {
|
||||
// New paragraph: end the current line, and leave one empty line per
|
||||
// blank source line (paragraph numbers count every newline).
|
||||
if (paragraph >= 0) {
|
||||
flush();
|
||||
for (let blank = paragraph + 1; blank < token.paragraph; blank++) flush();
|
||||
}
|
||||
paragraph = token.paragraph;
|
||||
}
|
||||
const wordWidth = font.measureText(token.text);
|
||||
const x = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
||||
if (current.spans.length > 0 && x + wordWidth > maxWidth) {
|
||||
flush();
|
||||
}
|
||||
const spanX = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
||||
current.spans.push({ tokenIndex, x: spanX, width: wordWidth, text: token.text });
|
||||
currentWidth = spanX + wordWidth;
|
||||
if (token.wordIndex >= 0) {
|
||||
wordLines[token.wordIndex] = lines.length;
|
||||
}
|
||||
}
|
||||
if (current.spans.length > 0 || lines.length === 0) flush();
|
||||
this.lines = lines;
|
||||
this.wordLines = wordLines;
|
||||
this.layoutKey = key;
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
|
||||
/**
|
||||
* The Teleprompter's recently opened scripts, most recent first, with the
|
||||
* word position the reader was at when it closed (so reopening resumes
|
||||
* there). Stored as a JSON array in the settings store.
|
||||
*/
|
||||
export type RecentScript = {
|
||||
path: string;
|
||||
name: string;
|
||||
/** Word index the reader was at when last closed; 0 = start. */
|
||||
position: number;
|
||||
openedMs: number;
|
||||
};
|
||||
|
||||
const RECENTS_KEY = "teleprompter.recents";
|
||||
const MAX_RECENTS = 12;
|
||||
|
||||
export function getRecentScripts(): RecentScript[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(getStringSetting(RECENTS_KEY, "[]"));
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed
|
||||
.filter((entry): entry is RecentScript => !!entry && typeof entry === "object" && typeof entry.path === "string")
|
||||
.map((entry) => ({
|
||||
path: entry.path,
|
||||
name: typeof entry.name === "string" && entry.name ? entry.name : baseName(entry.path),
|
||||
position: typeof entry.position === "number" && entry.position >= 0 ? entry.position : 0,
|
||||
openedMs: typeof entry.openedMs === "number" ? entry.openedMs : 0,
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function saveRecentScripts(recents: RecentScript[]): void {
|
||||
setStringSetting(RECENTS_KEY, JSON.stringify(recents.slice(0, MAX_RECENTS)));
|
||||
}
|
||||
|
||||
/** Move (or add) a script to the top of the list, keeping its saved position. */
|
||||
export function touchRecentScript(path: string, name: string): RecentScript {
|
||||
const recents = getRecentScripts();
|
||||
const existing = recents.find((entry) => entry.path === path);
|
||||
const entry: RecentScript = {
|
||||
path,
|
||||
name,
|
||||
position: existing?.position ?? 0,
|
||||
openedMs: Date.now(),
|
||||
};
|
||||
saveRecentScripts([entry, ...recents.filter((other) => other.path !== path)]);
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function saveRecentScriptPosition(path: string, position: number): void {
|
||||
const recents = getRecentScripts();
|
||||
const entry = recents.find((other) => other.path === path);
|
||||
if (!entry) return;
|
||||
entry.position = Math.max(0, position);
|
||||
saveRecentScripts(recents);
|
||||
}
|
||||
|
||||
export function removeRecentScript(path: string): void {
|
||||
saveRecentScripts(getRecentScripts().filter((entry) => entry.path !== path));
|
||||
}
|
||||
|
||||
export function clearRecentScripts(): void {
|
||||
saveRecentScripts([]);
|
||||
}
|
||||
|
||||
export function baseName(path: string): string {
|
||||
const slash = path.lastIndexOf("/");
|
||||
return slash >= 0 ? path.slice(slash + 1) : path;
|
||||
}
|
||||
|
||||
/** The last directory segment of a file path, for a dim "where it lives" hint. */
|
||||
export function parentDirName(path: string): string {
|
||||
const slash = path.lastIndexOf("/");
|
||||
if (slash <= 0) return "";
|
||||
return baseName(path.slice(0, slash));
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
/**
|
||||
* Follows a speaker through a script from live speech-to-text. Pure logic
|
||||
* (no NativeScript imports) so it runs under plain node for the tests.
|
||||
*
|
||||
* The script is tokenized once into display tokens; the subset with a
|
||||
* non-empty normalized key are the "words" the tracker aligns against. The
|
||||
* tracker keeps `position`: the index (in word space) of the next word the
|
||||
* speaker is expected to say. Each transcript update (replace semantics: the
|
||||
* whole current utterance, partial or final) re-aligns the tail of the
|
||||
* transcript against a window of script words around the current position,
|
||||
* with a local alignment that tolerates misrecognized, skipped and ad-libbed
|
||||
* words, and moves the position to just after the last aligned script word.
|
||||
*/
|
||||
|
||||
export type ScriptToken = {
|
||||
/** The token as written, for display. */
|
||||
text: string;
|
||||
/** Paragraph number (newline-separated), for layout. */
|
||||
paragraph: number;
|
||||
/** Lowercased alphanumeric key, or "" for punctuation-only tokens. */
|
||||
key: string;
|
||||
/** Index into the tracker's word list, or -1 when key is empty. */
|
||||
wordIndex: number;
|
||||
};
|
||||
|
||||
export type TokenizedScript = {
|
||||
tokens: ScriptToken[];
|
||||
/** Normalized keys of the trackable tokens, in order. */
|
||||
words: string[];
|
||||
/** Token index of each word. */
|
||||
wordTokens: number[];
|
||||
};
|
||||
|
||||
/** Lowercase, strip everything but letters/digits, fold common apostrophes. */
|
||||
export function normalizeWord(raw: string): string {
|
||||
return raw
|
||||
.normalize("NFKD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "");
|
||||
}
|
||||
|
||||
export function tokenizeScript(text: string): TokenizedScript {
|
||||
const tokens: ScriptToken[] = [];
|
||||
const words: string[] = [];
|
||||
const wordTokens: number[] = [];
|
||||
const paragraphs = text.replace(/\r/g, "").split("\n");
|
||||
for (let paragraph = 0; paragraph < paragraphs.length; paragraph++) {
|
||||
for (const raw of paragraphs[paragraph]!.split(/\s+/)) {
|
||||
if (!raw) continue;
|
||||
const key = normalizeWord(raw);
|
||||
const wordIndex = key ? words.length : -1;
|
||||
if (key) {
|
||||
words.push(key);
|
||||
wordTokens.push(tokens.length);
|
||||
}
|
||||
tokens.push({ text: raw, paragraph, key, wordIndex });
|
||||
}
|
||||
}
|
||||
return { tokens, words, wordTokens };
|
||||
}
|
||||
|
||||
/** Split a transcript into normalized keys, dropping empties. */
|
||||
export function transcriptKeys(text: string): string[] {
|
||||
return text
|
||||
.split(/\s+/)
|
||||
.map(normalizeWord)
|
||||
.filter((key) => key.length > 0);
|
||||
}
|
||||
|
||||
// Function words match so often that on their own they say little about
|
||||
// where the speaker is; they count for less than content words.
|
||||
const STOP_WORDS = new Set([
|
||||
"a", "an", "the", "and", "or", "but", "of", "to", "in", "on", "at", "for", "is", "it", "its",
|
||||
"i", "we", "you", "he", "she", "they", "that", "this", "as", "be", "are", "was", "were",
|
||||
"with", "by", "so", "if", "not", "no", "do", "have", "has", "had", "will", "can",
|
||||
]);
|
||||
|
||||
const MATCH_SCORE = 3;
|
||||
const STOP_WORD_MATCH_SCORE = 1.5;
|
||||
/** A fuzzy (edit-distance 1) or prefix match: real evidence, but weaker. */
|
||||
const FUZZY_MATCH_SCORE = 2;
|
||||
const MISMATCH_SCORE = -2;
|
||||
const GAP_SCORE = -1.5;
|
||||
/** The alignment must be worth at least this to move the position at all. */
|
||||
const MIN_ALIGNMENT_SCORE = 4.5;
|
||||
const MIN_MATCHED_WORDS = 2;
|
||||
/** Per-word cost of ending an alignment away from the expected position (a tie-breaker). */
|
||||
const FORWARD_DISTANCE_COST = 0.02;
|
||||
const BACKWARD_DISTANCE_COST = 0.1;
|
||||
|
||||
export type TrackerOptions = {
|
||||
/** Script words before the current position that stay in play (repeats, backtracking). */
|
||||
back?: number;
|
||||
/** Script words after the current position that stay in play (skipping ahead). */
|
||||
ahead?: number;
|
||||
/** Only the last this-many transcript words are aligned. */
|
||||
tailWords?: number;
|
||||
};
|
||||
|
||||
const DEFAULT_OPTIONS: Required<TrackerOptions> = { back: 30, ahead: 160, tailWords: 20 };
|
||||
|
||||
/**
|
||||
* Edit distance capped at 1 (true when the strings are within one edit),
|
||||
* which covers the typical STT slip: a dropped/extra letter or a substitution.
|
||||
*/
|
||||
export function withinOneEdit(a: string, b: string): boolean {
|
||||
if (a === b) return true;
|
||||
const lengthDelta = a.length - b.length;
|
||||
if (lengthDelta > 1 || lengthDelta < -1) return false;
|
||||
const [short, long] = a.length <= b.length ? [a, b] : [b, a];
|
||||
let i = 0;
|
||||
let j = 0;
|
||||
let edits = 0;
|
||||
while (i < short.length && j < long.length) {
|
||||
if (short[i] === long[j]) {
|
||||
i++;
|
||||
j++;
|
||||
continue;
|
||||
}
|
||||
if (++edits > 1) return false;
|
||||
if (short.length === long.length) {
|
||||
i++;
|
||||
}
|
||||
j++;
|
||||
}
|
||||
return edits + (long.length - j) <= 1;
|
||||
}
|
||||
|
||||
/** Score of aligning transcript key `spoken` with script key `expected`. */
|
||||
function matchScore(spoken: string, expected: string, isLastSpoken: boolean): number {
|
||||
if (spoken === expected) {
|
||||
return STOP_WORDS.has(expected) ? STOP_WORD_MATCH_SCORE : MATCH_SCORE;
|
||||
}
|
||||
// The final transcript word is often still being recognized: a prefix of
|
||||
// the script word counts, once it is long enough not to be noise.
|
||||
if (isLastSpoken && spoken.length >= 2 && expected.length > spoken.length && expected.startsWith(spoken)) {
|
||||
return FUZZY_MATCH_SCORE;
|
||||
}
|
||||
if (spoken.length >= 5 && expected.length >= 5 && withinOneEdit(spoken, expected)) {
|
||||
return FUZZY_MATCH_SCORE;
|
||||
}
|
||||
return MISMATCH_SCORE;
|
||||
}
|
||||
|
||||
export type AlignmentResult = {
|
||||
/** Index (in the script word list) of the word after the last aligned one; the new position. */
|
||||
position: number;
|
||||
score: number;
|
||||
matchedWords: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Align the transcript tail against script words [windowStart, windowEnd).
|
||||
* Local alignment (free start on both sides) that must consume the last
|
||||
* transcript word and may end anywhere in the window. Returns null when the
|
||||
* best alignment is too weak to trust.
|
||||
*/
|
||||
export function alignTail(
|
||||
words: readonly string[],
|
||||
spoken: readonly string[],
|
||||
windowStart: number,
|
||||
windowEnd: number,
|
||||
expectedPosition: number,
|
||||
): AlignmentResult | null {
|
||||
const m = spoken.length;
|
||||
const n = windowEnd - windowStart;
|
||||
if (m === 0 || n <= 0) return null;
|
||||
|
||||
// H[i][j]: best score of an alignment ending with spoken[i-1] / window[j-1]
|
||||
// (or a gap), clamped at 0 (free start). Matches counts how many spoken
|
||||
// words matched along the path that produced H[i][j].
|
||||
const width = n + 1;
|
||||
const score = new Float64Array((m + 1) * width);
|
||||
const matches = new Uint16Array((m + 1) * width);
|
||||
for (let i = 1; i <= m; i++) {
|
||||
const spokenWord = spoken[i - 1]!;
|
||||
const isLast = i === m;
|
||||
for (let j = 1; j <= n; j++) {
|
||||
const here = i * width + j;
|
||||
const s = matchScore(spokenWord, words[windowStart + j - 1]!, isLast);
|
||||
const diagonal = score[here - width - 1]! + s;
|
||||
const skipScript = score[here - 1]! + GAP_SCORE;
|
||||
const skipSpoken = score[here - width]! + GAP_SCORE;
|
||||
let best = 0;
|
||||
let bestMatches = 0;
|
||||
if (diagonal > best) {
|
||||
best = diagonal;
|
||||
bestMatches = matches[here - width - 1]! + (s > 0 ? 1 : 0);
|
||||
}
|
||||
if (skipScript > best) {
|
||||
best = skipScript;
|
||||
bestMatches = matches[here - 1]!;
|
||||
}
|
||||
if (skipSpoken > best) {
|
||||
best = skipSpoken;
|
||||
bestMatches = matches[here - width]!;
|
||||
}
|
||||
score[here] = best;
|
||||
matches[here] = bestMatches;
|
||||
}
|
||||
}
|
||||
|
||||
let bestColumn = -1;
|
||||
let bestAdjusted = -Infinity;
|
||||
for (let j = 1; j <= n; j++) {
|
||||
const raw = score[m * width + j]!;
|
||||
if (raw <= 0) continue;
|
||||
const position = windowStart + j;
|
||||
const distance = position - expectedPosition;
|
||||
const adjusted = raw - (distance >= 0 ? distance * FORWARD_DISTANCE_COST : -distance * BACKWARD_DISTANCE_COST);
|
||||
if (adjusted > bestAdjusted) {
|
||||
bestAdjusted = adjusted;
|
||||
bestColumn = j;
|
||||
}
|
||||
}
|
||||
if (bestColumn < 0) return null;
|
||||
const rawScore = score[m * width + bestColumn]!;
|
||||
const matchedWords = matches[m * width + bestColumn]!;
|
||||
if (rawScore < MIN_ALIGNMENT_SCORE || matchedWords < MIN_MATCHED_WORDS) return null;
|
||||
return { position: windowStart + bestColumn, score: rawScore, matchedWords };
|
||||
}
|
||||
|
||||
export class ScriptTracker {
|
||||
private readonly options: Required<TrackerOptions>;
|
||||
/** Index of the next word the speaker is expected to say (0..words.length). */
|
||||
position = 0;
|
||||
|
||||
constructor(
|
||||
private readonly words: readonly string[],
|
||||
options: TrackerOptions = {},
|
||||
) {
|
||||
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||
}
|
||||
|
||||
get wordCount(): number {
|
||||
return this.words.length;
|
||||
}
|
||||
|
||||
/** Jump the expected position (manual scroll, restart). */
|
||||
anchor(position: number): void {
|
||||
this.position = Math.max(0, Math.min(this.words.length, position));
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a transcript update (the whole current utterance, partial or
|
||||
* final). Returns true when the position moved.
|
||||
*/
|
||||
feed(text: string): boolean {
|
||||
const keys = transcriptKeys(text);
|
||||
if (keys.length === 0) return false;
|
||||
const tail = keys.length > this.options.tailWords ? keys.slice(-this.options.tailWords) : keys;
|
||||
const windowStart = Math.max(0, this.position - this.options.back);
|
||||
const windowEnd = Math.min(this.words.length, this.position + this.options.ahead);
|
||||
const result = alignTail(this.words, tail, windowStart, windowEnd, this.position);
|
||||
if (!result || result.position === this.position) return false;
|
||||
this.position = result.position;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { truncateText } from "../../graphics/textwrap";
|
||||
import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { LIST_ROW_TEXT_INSET } from "../../ui/metrics";
|
||||
import { MenuLayer, type MenuItem } from "../../ui/menu";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { appViewportSize } from "../../ui/shell/geometry";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
type InProcessAppOptions,
|
||||
type InProcessWindow,
|
||||
} from "../../ui/shell/in-process-window";
|
||||
import { FileBrowserLayer } from "../files/file-browser";
|
||||
import { TeleprompterReaderLayer } from "./reader";
|
||||
import {
|
||||
baseName,
|
||||
clearRecentScripts,
|
||||
getRecentScripts,
|
||||
parentDirName,
|
||||
removeRecentScript,
|
||||
saveRecentScriptPosition,
|
||||
touchRecentScript,
|
||||
} from "./recent-files";
|
||||
|
||||
export const TELEPROMPTER_WINDOW_ID = "teleprompter";
|
||||
export const TELEPROMPTER_SURFACE_ID = "window:teleprompter";
|
||||
|
||||
const SCRIPT_FILE = /\.(txt|md|markdown|text|log)$/i;
|
||||
|
||||
const HOME_LAYOUT = {
|
||||
x: 8,
|
||||
y: 8,
|
||||
width: 272,
|
||||
showBorder: false,
|
||||
minHeight: 0,
|
||||
maxHeight: appViewportSize("min").height - 16,
|
||||
opaque: true,
|
||||
};
|
||||
|
||||
export type TeleprompterAppOptions = InProcessAppOptions & {
|
||||
/** Hold continuous mic capture while a script is being tracked. */
|
||||
startContinuousVoiceCapture: () => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
appendLog: (message: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The Teleprompter app window. The home page is a Browse entry plus the
|
||||
* recently opened scripts; Browse pushes the Files app's browser to pick a
|
||||
* text file, and either path pushes the reader (see TeleprompterReaderLayer)
|
||||
* over the home page.
|
||||
*/
|
||||
export function createTeleprompterAppWindow(options: TeleprompterAppOptions): InProcessWindow {
|
||||
// The home menu's items are rebuilt in place whenever the recents change
|
||||
// (MenuLayer reads the array each paint).
|
||||
const homeItems: MenuItem[] = [];
|
||||
const home = new MenuLayer("Teleprompter", homeItems, HOME_LAYOUT);
|
||||
let created: InProcessWindow | null = null;
|
||||
let notice = "";
|
||||
|
||||
const requestRender = () => created?.requestRender();
|
||||
|
||||
/** The layer currently on top of the window's stack (the home menu at base). */
|
||||
const topLayer = (): Layer => {
|
||||
let top: Layer = home;
|
||||
created?.stack.topMatches((layer) => {
|
||||
top = layer;
|
||||
return true;
|
||||
});
|
||||
return top;
|
||||
};
|
||||
|
||||
const openScript = (ctx: LayerContext, path: string, name: string) => {
|
||||
const text = readTextFile(path);
|
||||
if (text === null) {
|
||||
removeRecentScript(path);
|
||||
notice = `Could not read ${name}`;
|
||||
options.appendLog(`teleprompter: could not read ${path}`);
|
||||
refreshHome();
|
||||
return;
|
||||
}
|
||||
const recent = touchRecentScript(path, name);
|
||||
notice = "";
|
||||
refreshHome();
|
||||
const layer = new TeleprompterReaderLayer(
|
||||
text,
|
||||
name,
|
||||
{
|
||||
requestRender,
|
||||
startCapture: () => options.startContinuousVoiceCapture(),
|
||||
stopCapture: () => options.stopContinuousVoiceCapture(),
|
||||
savePosition: (position) => saveRecentScriptPosition(path, position),
|
||||
},
|
||||
recent.position,
|
||||
);
|
||||
ctx.stack.push(layer);
|
||||
layer.attach();
|
||||
};
|
||||
|
||||
const openBrowser = (ctx: LayerContext) => {
|
||||
const stack = ctx.stack;
|
||||
const picker = new FileBrowserLayer({
|
||||
isSupportedFile: (name) => SCRIPT_FILE.test(name),
|
||||
onLeave: () => stack.pop(),
|
||||
onFilePicked: (entry: DirectoryEntry, pickCtx) => {
|
||||
if (!SCRIPT_FILE.test(entry.name)) {
|
||||
notice = "Pick a text file (.txt, .md)";
|
||||
return;
|
||||
}
|
||||
pickCtx.stack.pop();
|
||||
openScript(pickCtx, entry.path, entry.name);
|
||||
},
|
||||
});
|
||||
stack.push(picker);
|
||||
};
|
||||
|
||||
function refreshHome(): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const items: MenuItem[] = [
|
||||
{
|
||||
label: "Browse for a script...",
|
||||
onSelect: (ctx) => openBrowser(ctx),
|
||||
},
|
||||
];
|
||||
for (const recent of getRecentScripts()) {
|
||||
const folder = parentDirName(recent.path);
|
||||
items.push({
|
||||
label: recent.name || baseName(recent.path),
|
||||
onSelect: (ctx) => openScript(ctx, recent.path, recent.name),
|
||||
render: ({ image, x, y, width, selected, text }) => {
|
||||
const value = selected ? 255 : 200;
|
||||
const folderWidth = folder ? font.measureText(folder) : 0;
|
||||
const nameMax = folder ? width - folderWidth - 12 : width;
|
||||
image.drawText(font, x, y + LIST_ROW_TEXT_INSET, truncateText(font, text, Math.max(40, nameMax)), value);
|
||||
if (folder && nameMax >= 40) {
|
||||
image.drawText(font, x + width - folderWidth, y + LIST_ROW_TEXT_INSET, folder, 90);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
if (notice) {
|
||||
items.push({ label: notice, disabled: true, onSelect: () => {} });
|
||||
}
|
||||
homeItems.length = 0;
|
||||
homeItems.push(...items);
|
||||
// Land on the most recent script (one click resumes it), else on Browse.
|
||||
home.selectItem(getRecentScripts().length > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
refreshHome();
|
||||
|
||||
created = createInProcessWindow({
|
||||
appId: "teleprompter",
|
||||
windowId: TELEPROMPTER_WINDOW_ID,
|
||||
title: "Teleprompter",
|
||||
iconLetter: "Tp",
|
||||
icon: "scroll-text",
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
menuItems: () => {
|
||||
const top = topLayer();
|
||||
if (top instanceof TeleprompterReaderLayer) return top.buildMenuItems();
|
||||
if (top instanceof FileBrowserLayer) return top.buildMenuItems();
|
||||
if (top !== home) return [];
|
||||
return [
|
||||
{
|
||||
label: "Clear recent scripts",
|
||||
onSelect: (ctx) => {
|
||||
clearRecentScripts();
|
||||
refreshHome();
|
||||
ctx.stack.pop();
|
||||
},
|
||||
},
|
||||
];
|
||||
},
|
||||
onFocus: (lastInput) => {
|
||||
const top = topLayer();
|
||||
if (top instanceof FileBrowserLayer) top.onFocus(lastInput);
|
||||
},
|
||||
baseLayer: new YieldAtRootLayer(home),
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: options.onClosed,
|
||||
});
|
||||
return created;
|
||||
}
|
||||
@@ -36,6 +36,8 @@ export const ICON_SVGS = {
|
||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M12 18v4"/><path d="M2 14.499a5.5 5.5 0 0 0 9.591 3.675.6.6 0 0 1 .818.001A5.5 5.5 0 0 0 22 14.5c0-2.29-1.5-4-3-5.5l-5.492-5.312a2 2 0 0 0-3-.02L5 8.999c-1.5 1.5-3 3.2-3 5.5"/></svg>',
|
||||
terminal:
|
||||
'<svg viewBox="0 0 24 24" fill="none"><path d="m7 11 2-2-2-2"/><path d="M11 13h4"/><rect width="18" height="18" x="3" y="3" rx="2" ry="2"/></svg>',
|
||||
"scroll-text":
|
||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M15 12h-5"/><path d="M15 8h-5"/><path d="M19 17V5a2 2 0 0 0-2-2H4"/><path d="M8 21h12a2 2 0 0 0 2-2v-1a1 1 0 0 0-1-1H11a1 1 0 0 0-1 1v1a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v2a1 1 0 0 0 1 1h3"/></svg>',
|
||||
"file-text":
|
||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M6 22a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h8a2.4 2.4 0 0 1 1.704.706l3.588 3.588A2.4 2.4 0 0 1 20 8v12a2 2 0 0 1-2 2z"/><path d="M14 2v5a1 1 0 0 0 1 1h5"/><path d="M10 9H8"/><path d="M16 13H8"/><path d="M16 17H8"/></svg>',
|
||||
file:
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
// Pins the Teleprompter's speech-to-script tracker: tokenization, the fuzzy
|
||||
// word matcher, and the position tracking under realistic partial-transcript
|
||||
// streams (replace semantics), including misrecognitions, ad-libs, skipped
|
||||
// sentences and repeated phrases.
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
const tracker = require("../.test-build/app/apps/teleprompter/script-tracker.js");
|
||||
|
||||
const SCRIPT = `Good evening, everyone. Thank you for coming tonight.
|
||||
|
||||
Four score and seven years ago our fathers brought forth on this continent
|
||||
a new nation, conceived in liberty, and dedicated to the proposition that
|
||||
all men are created equal.
|
||||
|
||||
Now we are engaged in a great civil war, testing whether that nation, or any
|
||||
nation so conceived and so dedicated, can long endure. We are met on a great
|
||||
battle-field of that war.
|
||||
|
||||
We have come to dedicate a portion of that field, as a final resting place
|
||||
for those who here gave their lives that that nation might live.`;
|
||||
|
||||
/** Word index of the first occurrence of a normalized key, from `from`. */
|
||||
function wordAt(script, key, from = 0) {
|
||||
const index = script.words.indexOf(key, from);
|
||||
assert.notEqual(index, -1, `script has ${key}`);
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed an utterance the way a streaming recognizer delivers it: a growing
|
||||
* partial after every word, then the final. Returns the tracker position.
|
||||
*/
|
||||
function speak(t, utterance) {
|
||||
const words = utterance.split(/\s+/);
|
||||
for (let i = 1; i <= words.length; i++) {
|
||||
t.feed(words.slice(0, i).join(" "));
|
||||
}
|
||||
t.feed(utterance);
|
||||
return t.position;
|
||||
}
|
||||
|
||||
test("tokenizeScript separates display tokens from trackable words", () => {
|
||||
const script = tracker.tokenizeScript("Hello, world — it's 2026.\nNext line");
|
||||
assert.deepEqual(
|
||||
script.tokens.map((token) => token.text),
|
||||
["Hello,", "world", "—", "it's", "2026.", "Next", "line"],
|
||||
);
|
||||
assert.deepEqual(script.words, ["hello", "world", "its", "2026", "next", "line"]);
|
||||
assert.equal(script.tokens[2].wordIndex, -1);
|
||||
assert.equal(script.tokens[5].paragraph, 1);
|
||||
assert.deepEqual(script.wordTokens, [0, 1, 3, 4, 5, 6]);
|
||||
});
|
||||
|
||||
test("withinOneEdit accepts single slips and rejects more", () => {
|
||||
assert.ok(tracker.withinOneEdit("conceived", "conceived"));
|
||||
assert.ok(tracker.withinOneEdit("conceived", "concieved") === false, "transposition is two edits");
|
||||
assert.ok(tracker.withinOneEdit("liberty", "libery"));
|
||||
assert.ok(tracker.withinOneEdit("liberty", "libertyy"));
|
||||
assert.ok(tracker.withinOneEdit("liberty", "libarty"));
|
||||
assert.ok(!tracker.withinOneEdit("liberty", "library"));
|
||||
assert.ok(tracker.withinOneEdit("nation", "notion"), "one substitution");
|
||||
});
|
||||
|
||||
test("follows a clean read-through, word by word", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
assert.equal(t.position, 0);
|
||||
// A lone first word is not enough evidence to move.
|
||||
t.feed("Good");
|
||||
assert.equal(t.position, 0);
|
||||
t.feed("Good evening");
|
||||
assert.equal(t.position, 2);
|
||||
speak(t, "Good evening everyone thank you for coming tonight");
|
||||
assert.equal(t.position, wordAt(script, "four"));
|
||||
speak(t, "Four score and seven years ago our fathers brought forth on this continent");
|
||||
assert.equal(t.position, wordAt(script, "a", wordAt(script, "continent")));
|
||||
});
|
||||
|
||||
test("tolerates misrecognized and ad-libbed words", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
t.anchor(wordAt(script, "four"));
|
||||
// "score" heard as "scored", "fathers" as "father's", plus an inserted "um".
|
||||
speak(t, "for scored and seven years ago um our father's brought forth");
|
||||
assert.equal(t.position, wordAt(script, "on"));
|
||||
});
|
||||
|
||||
test("a partial last word counts as a prefix of the next script word", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
t.anchor(wordAt(script, "four"));
|
||||
t.feed("four score and seven years ago our fath");
|
||||
assert.equal(t.position, wordAt(script, "brought"));
|
||||
});
|
||||
|
||||
test("catches up when the speaker skips a sentence", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
t.anchor(wordAt(script, "four"));
|
||||
speak(t, "Now we are engaged in a great civil war");
|
||||
assert.equal(t.position, wordAt(script, "testing"));
|
||||
});
|
||||
|
||||
test("prefers the occurrence nearest the current position for repeated phrases", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
// "that nation" appears in paragraphs 3 and 4; anchored in paragraph 4 it
|
||||
// should pick the later one.
|
||||
const laterThat = wordAt(script, "gave");
|
||||
t.anchor(laterThat);
|
||||
speak(t, "their lives that that nation might live");
|
||||
assert.equal(t.position, script.words.length);
|
||||
});
|
||||
|
||||
test("does not move on transcripts that match nothing nearby", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words);
|
||||
t.anchor(wordAt(script, "four"));
|
||||
assert.equal(speak(t, "hang on let me find my glasses"), wordAt(script, "four"));
|
||||
});
|
||||
|
||||
test("manual anchoring re-bases the search window", () => {
|
||||
const script = tracker.tokenizeScript(SCRIPT);
|
||||
const t = new tracker.ScriptTracker(script.words, { ahead: 10, back: 5 });
|
||||
// Far outside the window from position 0.
|
||||
speak(t, "final resting place for those who here gave");
|
||||
assert.equal(t.position, 0);
|
||||
t.anchor(wordAt(script, "final"));
|
||||
speak(t, "final resting place for those who here gave");
|
||||
assert.equal(t.position, wordAt(script, "their"));
|
||||
});
|
||||
+2
-1
@@ -25,6 +25,7 @@
|
||||
"../app/apps/microphones/radar-dots.ts",
|
||||
"../app/apps/microphones/name-guess.ts",
|
||||
"../app/apps/evenhub/containers.ts",
|
||||
"../app/apps/timer/timer-model.ts"
|
||||
"../app/apps/timer/timer-model.ts",
|
||||
"../app/apps/teleprompter/script-tracker.ts"
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user