forked from Blendman974/faceclaw
Implement a basic voice assistant
This commit is contained in:
1 parent
d2c987d48a
commit
0aeb24ed00
14 files changed
+1923
-80
No files matched your search
@@ -0,0 +1,190 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { GESTURE_DOUBLE_CLICK } from "../gestures";
|
||||
import { drawSelectionHighlight } from "../menu";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
const DIALOG_X = 40;
|
||||
const DIALOG_Y = 40;
|
||||
const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
const MENU_ROW_H = 20;
|
||||
const MENU_ROWS = 2;
|
||||
|
||||
/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */
|
||||
type AssistantPhase = "thinking" | "done" | "error";
|
||||
|
||||
export type AssistantLayerCallbacks = {
|
||||
/** Record another utterance and continue this conversation. */
|
||||
onFollowUp: () => void;
|
||||
/** Abort the in-flight turn (double-click while thinking). */
|
||||
onCancel: () => void;
|
||||
/** The user closed the overlay (Done). */
|
||||
onClose: () => void;
|
||||
/** The layer left the stack by any path (Done, or the screen sleeping). */
|
||||
onRemoved?: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The assistant overlay: streamed reply text with a status line for tool
|
||||
* activity, and a Follow-up / Done menu once the turn ends. Double-click
|
||||
* cancels an in-flight turn. The shell owns the AssistantSession and drives
|
||||
* this layer's state through the on* methods as the turn streams.
|
||||
*/
|
||||
export class AssistantLayer implements Layer {
|
||||
private phase: AssistantPhase = "thinking";
|
||||
private replyText = "";
|
||||
private status = "Thinking...";
|
||||
private menuIndex = 0;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly callbacks: AssistantLayerCallbacks,
|
||||
) {}
|
||||
|
||||
/** Shell: a new turn (initial utterance or follow-up) is starting. */
|
||||
startTurn(): void {
|
||||
this.phase = "thinking";
|
||||
this.replyText = "";
|
||||
this.status = "Thinking...";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onTextDelta(_delta: string, textSoFar: string): void {
|
||||
this.replyText = textSoFar;
|
||||
if (this.phase === "thinking") this.status = "Thinking...";
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onToolActivity(label: string): void {
|
||||
if (this.phase === "thinking") this.status = `→ ${label}`;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onTurnDone(): void {
|
||||
this.phase = "done";
|
||||
this.status = this.replyText.trim() ? "" : "(no reply)";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onError(message: string): void {
|
||||
this.phase = "error";
|
||||
this.status = message;
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
// Solid dialog box (fill 1, matching the voice dialog: opaque after 4bpp
|
||||
// quantization, but not the color-key transparent 0).
|
||||
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
|
||||
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
|
||||
|
||||
const left = DIALOG_X + 16;
|
||||
image.drawText(font, left, DIALOG_Y + 12, this.phase === "thinking" ? "Assistant ●" : "Assistant", 220);
|
||||
if (this.status) {
|
||||
const statusValue = this.phase === "error" ? 200 : 130;
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), statusValue);
|
||||
}
|
||||
|
||||
const inMenu = this.phase !== "thinking";
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const textTop = DIALOG_Y + 56;
|
||||
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
|
||||
|
||||
const wrapped = wrapText(font, this.replyText, TEXT_MAX_WIDTH);
|
||||
// Keep the tail visible as text streams past the bottom.
|
||||
const firstLine = Math.max(0, wrapped.length - maxLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
}
|
||||
|
||||
if (inMenu) {
|
||||
const rows = ["Follow-up", "Done"];
|
||||
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const rowY = menuTop + i * MENU_ROW_H;
|
||||
const selected = i === this.menuIndex;
|
||||
if (selected) {
|
||||
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
|
||||
}
|
||||
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
|
||||
}
|
||||
} else {
|
||||
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
if (this.phase === "thinking") {
|
||||
if (event.type === "double-click") {
|
||||
this.callbacks.onCancel();
|
||||
this.phase = "done";
|
||||
this.status = "Cancelled";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
return;
|
||||
}
|
||||
// done / error: the Follow-up / Done menu.
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "click":
|
||||
if (this.menuIndex === 0) {
|
||||
this.callbacks.onFollowUp();
|
||||
} else {
|
||||
this.callbacks.onClose();
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
this.callbacks.onClose();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.callbacks.onRemoved?.();
|
||||
}
|
||||
}
|
||||
|
||||
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) lines.push(line);
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncate(font: BdfFont, text: string, maxWidth: number): string {
|
||||
if (font.measureText(text) <= maxWidth) return text;
|
||||
let out = text;
|
||||
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
|
||||
out = out.slice(0, -1);
|
||||
}
|
||||
return `${out}...`;
|
||||
}
|
||||
+136
-42
@@ -13,19 +13,47 @@ const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
const MENU_ROW_H = 20;
|
||||
const MENU_ROWS = 3;
|
||||
// After the mic stops, the provider's final transcript can trail in (cloud
|
||||
// commit round-trip); wait this long for it before refining with what we have.
|
||||
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
|
||||
|
||||
/**
|
||||
* The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or
|
||||
* click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing"
|
||||
* click-to-finish); "menu" is the send / Continue / Discard menu; "continuing"
|
||||
* records a follow-up utterance (always click-to-finish); "refining" streams
|
||||
* the LLM-merged text, then returns to "menu".
|
||||
*/
|
||||
type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
|
||||
|
||||
/**
|
||||
* A destination the captured text can be sent to. The shell supplies these per
|
||||
* entry point: "Send to Assistant" (hand off to the voice assistant) and/or
|
||||
* "Type Into App" (deliver to the foreground window's text input).
|
||||
*/
|
||||
export type VoiceSendTarget = {
|
||||
id: string;
|
||||
label: string;
|
||||
onSend: (text: string) => void;
|
||||
};
|
||||
|
||||
export type VoiceInputLayerOptions = {
|
||||
actions: LayerActions;
|
||||
/** Post-removal cleanup (also fires when the screen turns off). */
|
||||
onClosed: () => void;
|
||||
/** Pop this layer off the shell stack. */
|
||||
dismiss: () => void;
|
||||
/** Ordered send destinations shown as the first menu rows. */
|
||||
sendTargets: VoiceSendTarget[];
|
||||
/** Which send target is highlighted by default (entry-point dependent). */
|
||||
defaultTargetIndex?: number;
|
||||
/** A single click ends the utterance (opened from a menu, no held button). */
|
||||
finishOnClick?: boolean;
|
||||
/** Wakeword flow: mic starts immediately and ends on detected silence. */
|
||||
handsFree?: boolean;
|
||||
/** Skip the menu: on capture end, send straight to the default target. */
|
||||
autoSend?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
|
||||
* The mic runs while the button is held (long-press); releasing it stops the
|
||||
@@ -58,17 +86,34 @@ export class VoiceInputLayer implements Layer {
|
||||
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private refineHandle: AnthropicStreamHandle | null = null;
|
||||
private menuIndex = 0;
|
||||
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
|
||||
private pendingAutoSend = false;
|
||||
private autoSendTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeSpeechEnd: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
private readonly onSend: (text: string) => void,
|
||||
private readonly finishOnClick = false,
|
||||
private readonly handsFree = false,
|
||||
) {}
|
||||
private readonly actions: LayerActions;
|
||||
private readonly onClosed: () => void;
|
||||
private readonly dismiss: () => void;
|
||||
private readonly sendTargets: VoiceSendTarget[];
|
||||
private readonly defaultTargetIndex: number;
|
||||
private readonly finishOnClick: boolean;
|
||||
private readonly handsFree: boolean;
|
||||
private readonly autoSend: boolean;
|
||||
|
||||
constructor(options: VoiceInputLayerOptions) {
|
||||
this.actions = options.actions;
|
||||
this.onClosed = options.onClosed;
|
||||
this.dismiss = options.dismiss;
|
||||
this.sendTargets = options.sendTargets;
|
||||
this.finishOnClick = options.finishOnClick ?? false;
|
||||
this.handsFree = options.handsFree ?? false;
|
||||
this.autoSend = options.autoSend ?? false;
|
||||
const defaultIndex = options.defaultTargetIndex ?? 0;
|
||||
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
|
||||
this.menuIndex = this.defaultTargetIndex;
|
||||
}
|
||||
|
||||
startCapture(): void {
|
||||
if (this.capturing) return;
|
||||
@@ -113,13 +158,66 @@ export class VoiceInputLayer implements Layer {
|
||||
this.capturing = false;
|
||||
this.phase = "menu";
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.status.startsWith("Listening")) {
|
||||
if (this.autoSend && this.sendTargets.length) {
|
||||
// Skip-confirmation (wakeword): send to the default target as soon as the
|
||||
// transcript finalizes, or after a short wait for the trailing final.
|
||||
this.pendingAutoSend = true;
|
||||
this.status = "Sending...";
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
} else if (this.status.startsWith("Listening")) {
|
||||
this.status = "Send, continue, or discard?";
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
/** Fire the queued skip-confirmation send (or fall back to the menu). */
|
||||
private performAutoSend(): void {
|
||||
if (this.autoSendTimer !== null) {
|
||||
clearTimeout(this.autoSendTimer);
|
||||
this.autoSendTimer = null;
|
||||
}
|
||||
if (!this.pendingAutoSend) return;
|
||||
this.pendingAutoSend = false;
|
||||
const text = this.displayText().trim();
|
||||
const target = this.sendTargets[this.defaultTargetIndex];
|
||||
if (text && target) {
|
||||
this.dismiss();
|
||||
target.onSend(text);
|
||||
} else {
|
||||
// Nothing heard; fall back to the menu so the user can retry or discard.
|
||||
this.status = "Send, continue, or discard?";
|
||||
this.actions.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
/** The menu rows: one per send target, then Continue, then Discard. */
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
rows.push({
|
||||
label: target.label,
|
||||
dim: !hasText,
|
||||
onSelect: () => {
|
||||
this.dismiss();
|
||||
if (hasText) target.onSend(text);
|
||||
},
|
||||
});
|
||||
}
|
||||
rows.push({
|
||||
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
},
|
||||
});
|
||||
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
|
||||
return rows;
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
@@ -133,9 +231,9 @@ export class VoiceInputLayer implements Layer {
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
|
||||
|
||||
const inMenu = this.phase === "menu";
|
||||
const hasText = this.displayText().trim().length > 0;
|
||||
// Leave room for the three-row menu when it is showing.
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const rows = inMenu ? this.menuRows() : [];
|
||||
// Reserve space for the actual number of rows this menu has.
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const textTop = DIALOG_Y + 56;
|
||||
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
|
||||
|
||||
@@ -147,14 +245,7 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
if (inMenu) {
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean }> = [
|
||||
// Dim "Send" when there is nothing to send.
|
||||
{ label: "Send", dim: !hasText },
|
||||
{ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey },
|
||||
{ label: "Discard", dim: false },
|
||||
];
|
||||
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
|
||||
const menuTop = DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 2;
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const rowY = menuTop + i * MENU_ROW_H;
|
||||
const selected = i === this.menuIndex;
|
||||
@@ -170,11 +261,11 @@ export class VoiceInputLayer implements Layer {
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
switch (this.phase) {
|
||||
case "capturing":
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
this.dismiss();
|
||||
} else if (this.clickEndsCapture && event.type === "click") {
|
||||
this.endCapture();
|
||||
}
|
||||
@@ -192,37 +283,29 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
return;
|
||||
case "menu":
|
||||
this.handleMenuInput(event, ctx);
|
||||
this.handleMenuInput(event);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
private handleMenuInput(event: DashboardInputEvent): void {
|
||||
const rowCount = this.menuRows().length;
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
|
||||
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
|
||||
this.menuIndex = (this.menuIndex + 1) % rowCount;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "click":
|
||||
if (this.menuIndex === 0) {
|
||||
const text = this.displayText().trim();
|
||||
if (text) this.onSend(text);
|
||||
ctx.stack.pop();
|
||||
} else if (this.menuIndex === 1) {
|
||||
if (anthropicApiKeySetting.get().trim()) {
|
||||
this.startContinuation();
|
||||
}
|
||||
// No key: the row is disabled; the click does nothing.
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
case "click": {
|
||||
const row = this.menuRows()[this.menuIndex];
|
||||
row?.onSelect();
|
||||
return;
|
||||
}
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
this.dismiss();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
@@ -323,6 +406,11 @@ export class VoiceInputLayer implements Layer {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
}
|
||||
if (this.autoSendTimer !== null) {
|
||||
clearTimeout(this.autoSendTimer);
|
||||
this.autoSendTimer = null;
|
||||
}
|
||||
this.pendingAutoSend = false;
|
||||
this.refineHandle?.cancel();
|
||||
this.refineHandle = null;
|
||||
if (this.capturing) {
|
||||
@@ -380,6 +468,12 @@ export class VoiceInputLayer implements Layer {
|
||||
this.beginRefine();
|
||||
return;
|
||||
}
|
||||
if (this.pendingAutoSend) {
|
||||
// The utterance finalized; skip-confirmation can fire without waiting
|
||||
// out the fallback timer.
|
||||
this.performAutoSend();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
|
||||
@@ -294,6 +294,13 @@ export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
id: "assistant-skip-confirmation",
|
||||
label: "Send to assistant without confirming",
|
||||
storageKey: "assistant.skipConfirmationAfterWakeword",
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
id: "elevenlabs-api-key",
|
||||
label: "ElevenLabs key",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { getDashboardLogo } from "../../graphics/logo";
|
||||
import { shell } from "../shell/shell";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
brightnessSetting,
|
||||
elevenLabsApiKeySetting,
|
||||
@@ -56,6 +57,9 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
// When on, a wakeword utterance goes straight to the assistant with no
|
||||
// Send/Type menu step.
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
+291
-14
@@ -1,11 +1,17 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
|
||||
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
||||
import { DashboardInputEvent, LayerActions, LayerStack, noopLayerActions } from "../layers";
|
||||
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
||||
import { MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer } from "../apps/voice-input";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
|
||||
import { AssistantLayer } from "../apps/assistant";
|
||||
import { AssistantSession } from "../../assistant/session";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
onAnySettingChanged,
|
||||
timeFormatSetting,
|
||||
@@ -99,6 +105,64 @@ class ShellOverlayMenuLayer extends MenuLayer {
|
||||
}
|
||||
}
|
||||
|
||||
/** How long a show_alert popup stays before auto-dismissing. */
|
||||
const ALERT_DISMISS_MS = 6000;
|
||||
const ALERT_X = 40;
|
||||
const ALERT_W = G2_LENS_WIDTH - 80;
|
||||
const ALERT_Y = 96;
|
||||
const ALERT_H = 96;
|
||||
|
||||
/**
|
||||
* A brief text popup on the shell surface (the assistant's show_alert tool and
|
||||
* other short notices). Auto-dismisses after a few seconds; a click or
|
||||
* double-click dismisses it early.
|
||||
*/
|
||||
class ShellAlertLayer implements Layer {
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(private readonly text: string, private readonly onDismiss: () => void) {
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null;
|
||||
this.onDismiss();
|
||||
}, ALERT_DISMISS_MS);
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const image = paintBelow();
|
||||
const font = getDefaultSmallFont();
|
||||
image.fillRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 1, 10);
|
||||
image.drawRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 90, 10);
|
||||
image.drawText(font, ALERT_X + 16, ALERT_Y + 12, "Assistant", 200);
|
||||
image.drawTextWrapped({
|
||||
font,
|
||||
x: ALERT_X + 16,
|
||||
y: ALERT_Y + 34,
|
||||
width: ALERT_W - 32,
|
||||
text: this.text,
|
||||
value: 235,
|
||||
});
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
if (event.type === "click" || event.type === "double-click") {
|
||||
this.clearTimer();
|
||||
this.onDismiss();
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.clearTimer();
|
||||
}
|
||||
|
||||
private clearTimer(): void {
|
||||
if (this.timer !== null) {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class Shell {
|
||||
private windows: ShellWindow[] = [];
|
||||
private selectedIndex = 0;
|
||||
@@ -111,6 +175,8 @@ class Shell {
|
||||
// notification icons and the battery indicators.
|
||||
private readonly trayIcons = new Map<string, GrayImage>();
|
||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||
private assistantSession: AssistantSession | null = null;
|
||||
private assistantLayer: AssistantLayer | null = null;
|
||||
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private readonly actions: LayerActions = { ...noopActions };
|
||||
private config: ShellConfig = {
|
||||
@@ -236,6 +302,21 @@ class Shell {
|
||||
return this.screenOn;
|
||||
}
|
||||
|
||||
/** Current headset battery levels (for the assistant's get_state tool). */
|
||||
getBatteryLevels(): ShellChromeState["battery"] {
|
||||
return this.battery;
|
||||
}
|
||||
|
||||
/**
|
||||
* The foreground app, or null when only the launcher is showing. The launcher
|
||||
* is a pinned window but counts as "no app" for the assistant's context.
|
||||
*/
|
||||
getForegroundApp(): { appId: string; title: string } | null {
|
||||
const window = this.foregroundWindow();
|
||||
if (!window || window.appId === "launcher") return null;
|
||||
return { appId: window.appId, title: window.title };
|
||||
}
|
||||
|
||||
noteUserActivity(nowMs = Date.now()): void {
|
||||
this.lastInputAtMs = nowMs;
|
||||
}
|
||||
@@ -283,7 +364,10 @@ class Shell {
|
||||
// An open voice dialog suspends the timeout: a long dictation or refine
|
||||
// has no button presses, but the screen must stay on for it. Sliding
|
||||
// lastInputAtMs forward also restarts the full timeout when it closes.
|
||||
if (this.activeVoiceLayer) {
|
||||
// An in-flight assistant turn suspends it for the same reason (a tool loop
|
||||
// can run for a while with no input); once the turn ends and the
|
||||
// Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -358,7 +442,13 @@ class Shell {
|
||||
this.lastInputAtMs = Date.now();
|
||||
const wokeScreen = !this.screenOn && this.wake("sidebar");
|
||||
if (action === "voice-input" && !this.activeVoiceLayer) {
|
||||
this.openVoiceDialog(false, true);
|
||||
if (this.assistantLayer) {
|
||||
// The assistant overlay is up; a wakeword continues that conversation.
|
||||
this.startAssistantFollowUp(true);
|
||||
} else {
|
||||
// Wakeword defaults the highlight to Send to Assistant.
|
||||
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
|
||||
}
|
||||
}
|
||||
return { shell: wokeScreen || action === "voice-input", window: false };
|
||||
}
|
||||
@@ -489,25 +579,77 @@ class Shell {
|
||||
next?.requestRender();
|
||||
}
|
||||
|
||||
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
|
||||
const layer = new VoiceInputLayer(
|
||||
this.config.actions,
|
||||
() => {
|
||||
private openVoiceDialog(options: {
|
||||
finishOnClick?: boolean;
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
const targets = this.buildVoiceSendTargets();
|
||||
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
|
||||
if (defaultIndex < 0) defaultIndex = 0;
|
||||
// Skip the menu only for a hands-free (wakeword) capture aimed at the
|
||||
// assistant, when the user has opted into it.
|
||||
const autoSend =
|
||||
Boolean(options.handsFree) &&
|
||||
options.defaultTarget === "assistant" &&
|
||||
targets[defaultIndex]?.id === "assistant" &&
|
||||
assistantSkipConfirmationSetting.get();
|
||||
|
||||
const layer = new VoiceInputLayer({
|
||||
actions: this.config.actions,
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === layer) {
|
||||
this.activeVoiceLayer = null;
|
||||
// The idle countdown restarts in full once voice input ends.
|
||||
this.noteUserActivity();
|
||||
}
|
||||
},
|
||||
(text) => this.sendTextToForegroundWindow(text),
|
||||
finishOnClick,
|
||||
handsFree,
|
||||
);
|
||||
dismiss: () => {
|
||||
this.stack.popIfTop((top) => top === layer);
|
||||
},
|
||||
sendTargets: targets,
|
||||
defaultTargetIndex: defaultIndex,
|
||||
finishOnClick: options.finishOnClick ?? false,
|
||||
handsFree: options.handsFree ?? false,
|
||||
autoSend,
|
||||
});
|
||||
this.activeVoiceLayer = layer;
|
||||
this.stack.push(layer);
|
||||
layer.startCapture();
|
||||
}
|
||||
|
||||
/**
|
||||
* The send destinations offered by the voice dialog: the assistant (when an
|
||||
* API key is configured) and/or typing into the foreground window (when it
|
||||
* accepts text). Order fixes the menu row order.
|
||||
*/
|
||||
private buildVoiceSendTargets(): VoiceSendTarget[] {
|
||||
const targets: VoiceSendTarget[] = [];
|
||||
if (this.isAssistantAvailable()) {
|
||||
targets.push({
|
||||
id: "assistant",
|
||||
label: "Send to Assistant",
|
||||
onSend: (text) => this.sendToAssistant(text),
|
||||
});
|
||||
}
|
||||
if (this.foregroundWindow()?.receiveTextInput) {
|
||||
targets.push({
|
||||
id: "app",
|
||||
label: "Type Into App",
|
||||
onSend: (text) => this.sendTextToForegroundWindow(text),
|
||||
});
|
||||
}
|
||||
// Guarantee at least one destination so the dialog is never a dead end.
|
||||
if (targets.length === 0) {
|
||||
targets.push({
|
||||
id: "app",
|
||||
label: "Type Into App",
|
||||
onSend: (text) => this.sendTextToForegroundWindow(text),
|
||||
});
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
|
||||
sendTextToForegroundWindow(text: string): void {
|
||||
this.foregroundWindow()?.receiveTextInput?.(text);
|
||||
@@ -521,9 +663,130 @@ class Shell {
|
||||
*/
|
||||
startVoiceInput(): void {
|
||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
||||
// The transcript is aimed at the window whose menu requested it.
|
||||
// The transcript is aimed at the window whose menu requested it; the menu
|
||||
// entry point defaults the highlight to Type Into App.
|
||||
this.focus = "window";
|
||||
this.openVoiceDialog(true);
|
||||
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private isAssistantAvailable(): boolean {
|
||||
return anthropicApiKeySetting.get().trim().length > 0;
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const apiKey = anthropicApiKeySetting.get().trim();
|
||||
if (!apiKey) return null;
|
||||
if (!this.assistantSession || this.assistantSession.isExpired()) {
|
||||
this.assistantSession = new AssistantSession(apiKey);
|
||||
}
|
||||
return this.assistantSession;
|
||||
}
|
||||
|
||||
private buildAssistantContext(): AssistantContext {
|
||||
const foreground = this.getForegroundApp();
|
||||
return {
|
||||
foregroundApp: foreground?.appId ?? null,
|
||||
foregroundTitle: foreground?.title ?? null,
|
||||
screenOn: this.screenOn,
|
||||
localTime: formatAssistantTime(new Date()),
|
||||
headsetBattery: this.battery.headset,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start (or continue) an assistant conversation from a finalized utterance.
|
||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||
* existing session and overlay.
|
||||
*/
|
||||
sendToAssistant(text: string): void {
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert("Set an Anthropic API key in Settings to use the assistant.");
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
let layer = this.assistantLayer;
|
||||
if (!layer) {
|
||||
const created = new AssistantLayer(this.config.actions, {
|
||||
onFollowUp: () => this.startAssistantFollowUp(),
|
||||
onCancel: () => this.assistantSession?.cancel(),
|
||||
onClose: () => this.closeAssistantLayer(),
|
||||
onRemoved: () => {
|
||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||
// stop the turn and drop the reference so a later query starts clean.
|
||||
this.assistantSession?.cancel();
|
||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||
},
|
||||
});
|
||||
layer = created;
|
||||
this.assistantLayer = created;
|
||||
this.stack.push(created);
|
||||
}
|
||||
this.runAssistantTurn(session, layer, text);
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
||||
layer.startTurn();
|
||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
|
||||
onToolActivity: (label) => layer.onToolActivity(label),
|
||||
onTurnDone: () => layer.onTurnDone(),
|
||||
onError: (message) => layer.onError(message),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record another utterance in the current assistant conversation. handsFree
|
||||
* (from a wakeword over the overlay) starts the mic immediately; otherwise
|
||||
* (the Follow-up menu button) a click ends the utterance.
|
||||
*/
|
||||
private startAssistantFollowUp(handsFree = false): void {
|
||||
const layer = this.assistantLayer;
|
||||
const session = this.assistantSession;
|
||||
if (!layer || !session || this.activeVoiceLayer) return;
|
||||
const voice = new VoiceInputLayer({
|
||||
actions: this.config.actions,
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === voice) {
|
||||
this.activeVoiceLayer = null;
|
||||
this.noteUserActivity();
|
||||
}
|
||||
},
|
||||
dismiss: () => {
|
||||
this.stack.popIfTop((top) => top === voice);
|
||||
},
|
||||
sendTargets: [
|
||||
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
|
||||
],
|
||||
finishOnClick: !handsFree,
|
||||
handsFree,
|
||||
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
|
||||
});
|
||||
this.activeVoiceLayer = voice;
|
||||
this.stack.push(voice);
|
||||
voice.startCapture();
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private closeAssistantLayer(): void {
|
||||
// Popping fires the layer's onRemoved, which cancels the turn and clears
|
||||
// this.assistantLayer.
|
||||
const layer = this.assistantLayer;
|
||||
if (layer) this.stack.popIfTop((top) => top === layer);
|
||||
this.noteUserActivity();
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
|
||||
showAlert(text: string): void {
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
const layer = new ShellAlertLayer(text, () => {
|
||||
this.stack.popIfTop((top) => top === layer);
|
||||
this.config.requestShellRender();
|
||||
});
|
||||
this.stack.push(layer);
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
@@ -586,6 +849,20 @@ class Shell {
|
||||
|
||||
export const shell = new Shell();
|
||||
|
||||
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
/** Human-readable local time for the assistant's per-turn context. */
|
||||
function formatAssistantTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
return `${day}, ${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
|
||||
if (event.kind === "sys-event") {
|
||||
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
|
||||
|
||||
Reference in new issue
Block a user