Add initial voice-command support

Adds a single wakeword-style voice command, "screen on", and deals with
all the wiring and permissions that entails. Currently this only
supports using the phone's mic (or a bluetooth headset), not the G2 mic.
This commit is contained in:
James Babcock committed 2026-04-27 16:45:09 -07:00
1 parent c4cb4f9f01
commit 9fbb985db6
35 files changed
+1722 -16

No files matched your search

+25 -11
View File
@@ -1,7 +1,9 @@
import { Application, Utils } from "@nativescript/core";
const PERMISSION_REQUEST_CODE = 4242;
const VOICE_PERMISSION_REQUEST_CODE = 4243;
const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS";
const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO";
function getActivity(): androidx.appcompat.app.AppCompatActivity {
const activity = Application.android.foregroundActivity ?? Application.android.startActivity;
@@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] {
return permissions;
}
export async function ensureBlePermissions(): Promise<void> {
function isPermissionGranted(permission: string): boolean {
const context = getContext();
return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) ===
android.content.pm.PackageManager.PERMISSION_GRANTED;
}
async function ensurePermissions(
permissions: string[],
requestCode: number,
denialLabel: string,
): Promise<void> {
if (!global.isAndroid) return;
const context = getContext();
const activity = getActivity();
const required = getRequiredPermissions();
const missing = required.filter(
(permission) =>
androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !==
android.content.pm.PackageManager.PERMISSION_GRANTED,
);
const missing = permissions.filter((permission) => !isPermissionGranted(permission));
if (missing.length === 0) return;
@@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise<void> {
permissions: string[];
grantResults: number[];
}) => {
if (args.requestCode !== PERMISSION_REQUEST_CODE) return;
if (args.requestCode !== requestCode) return;
Application.android.off(Application.android.activityRequestPermissionsEvent, callback);
const denied = missing.filter(
@@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise<void> {
);
if (denied.length > 0) {
reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`));
reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`));
return;
}
resolve();
@@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise<void> {
androidx.core.app.ActivityCompat.requestPermissions(
activity,
toJavaStringArray(missing),
PERMISSION_REQUEST_CODE,
requestCode,
);
});
}
export async function ensureBlePermissions(): Promise<void> {
await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth");
}
export async function ensureVoicePermissions(): Promise<void> {
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
}
+73 -1
View File
@@ -1,19 +1,21 @@
import { ImageSource } from "@nativescript/core";
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses";
import { ensureBlePermissions } from "./android-permissions";
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { voiceControlBridge } from "../native/voice-control";
import {
applyDashboardScreenTimeout,
consumeDashboardTiledWakePaint,
drawDashboard,
getDashboardNightscoutSettings,
getDashboardSystemCardName,
isDashboardVoiceControlEnabled,
receiveInput,
resetDashboardSleepTimerAndWake,
setDashboardBatteryLevels,
@@ -21,6 +23,7 @@ import {
setDashboardNightscoutApiToken,
setDashboardNightscoutSiteUrl,
setDashboardSystemCardName,
setDashboardVoiceControlEnabled,
} from "../ui/dashboard";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -118,6 +121,8 @@ class DashboardController {
private offFrameMetrics: (() => void) | null = null;
private offMedia: (() => void) | null = null;
private offNightscout: (() => void) | null = null;
private offVoiceStatus: (() => void) | null = null;
private offVoiceWakeWord: (() => void) | null = null;
private lastInput = "waiting...";
private lastSys = "none yet";
private renderInProgress = false;
@@ -132,6 +137,7 @@ class DashboardController {
startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(),
startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(),
endTextSettingEdit: () => this.endTextSettingEdit(),
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
});
}
@@ -322,11 +328,20 @@ class DashboardController {
});
}
});
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
this.appendLog(state.status);
});
this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
void this.handleWakeWord(keyword).catch((error) => {
this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
});
});
await mediaControllerBridge.start();
await nightscoutBridge.start();
await communicator.start();
await this.requestRender("initial");
this.startVoiceControlIfEnabled();
this.dashboardTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
@@ -360,8 +375,13 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
if (communicator) {
await communicator.close().catch(() => {});
}
@@ -397,6 +417,10 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
const communicator = this.communicator;
this.communicator = null;
@@ -413,6 +437,7 @@ class DashboardController {
}
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
await communicator?.close().catch(() => {});
} finally {
stopForegroundNotification();
@@ -445,6 +470,30 @@ class DashboardController {
this.emit();
}
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
if (enabled) {
try {
await ensureVoicePermissions();
} catch (error) {
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
return;
}
}
setDashboardVoiceControlEnabled(enabled);
this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
if (enabled) {
this.startVoiceControlIfEnabled();
} else {
voiceControlBridge.stop();
}
if (this.phase === "connected" && this.communicator) {
await this.requestRender("interval").catch((error) => {
this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
});
}
}
private endTextSettingEdit(): void {
const finishedKind = this.activeTextSettingEditorKind;
this.editingSystemCardName = false;
@@ -580,6 +629,29 @@ class DashboardController {
}
}
private async handleWakeWord(keyword: string): Promise<void> {
const normalized = keyword.trim();
if (normalized.length === 0) return;
this.appendLog(`wake-word detected: ${normalized}`);
const changed = resetDashboardSleepTimerAndWake();
if (this.phase === "connected" && this.communicator) {
if (changed) {
await this.requestRender("initial");
}
return;
}
if (changed) {
const image = drawDashboard();
this.setDisplayPreview(grayImageToPreviewSource(image));
}
}
private startVoiceControlIfEnabled(): void {
if (!this.communicator) return;
if (!isDashboardVoiceControlEnabled()) return;
voiceControlBridge.start();
}
private clearDashboardTimer(): void {
if (this.dashboardTimer) {
clearInterval(this.dashboardTimer);