forked from Blendman974/faceclaw
Add initial voice-command support
Adds a single wakeword-style voice command, "screen on", and deals with all the wiring and permissions that entails. Currently this only supports using the phone's mic (or a bluetooth headset), not the G2 mic.
This commit is contained in:
1 parent
c4cb4f9f01
commit
9fbb985db6
35 files changed
+1722
-16
No files matched your search
@@ -1,7 +1,9 @@
|
||||
import { Application, Utils } from "@nativescript/core";
|
||||
|
||||
const PERMISSION_REQUEST_CODE = 4242;
|
||||
const VOICE_PERMISSION_REQUEST_CODE = 4243;
|
||||
const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS";
|
||||
const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO";
|
||||
|
||||
function getActivity(): androidx.appcompat.app.AppCompatActivity {
|
||||
const activity = Application.android.foregroundActivity ?? Application.android.startActivity;
|
||||
@@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] {
|
||||
return permissions;
|
||||
}
|
||||
|
||||
export async function ensureBlePermissions(): Promise<void> {
|
||||
function isPermissionGranted(permission: string): boolean {
|
||||
const context = getContext();
|
||||
return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) ===
|
||||
android.content.pm.PackageManager.PERMISSION_GRANTED;
|
||||
}
|
||||
|
||||
async function ensurePermissions(
|
||||
permissions: string[],
|
||||
requestCode: number,
|
||||
denialLabel: string,
|
||||
): Promise<void> {
|
||||
if (!global.isAndroid) return;
|
||||
|
||||
const context = getContext();
|
||||
const activity = getActivity();
|
||||
const required = getRequiredPermissions();
|
||||
const missing = required.filter(
|
||||
(permission) =>
|
||||
androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !==
|
||||
android.content.pm.PackageManager.PERMISSION_GRANTED,
|
||||
);
|
||||
const missing = permissions.filter((permission) => !isPermissionGranted(permission));
|
||||
|
||||
if (missing.length === 0) return;
|
||||
|
||||
@@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise<void> {
|
||||
permissions: string[];
|
||||
grantResults: number[];
|
||||
}) => {
|
||||
if (args.requestCode !== PERMISSION_REQUEST_CODE) return;
|
||||
if (args.requestCode !== requestCode) return;
|
||||
Application.android.off(Application.android.activityRequestPermissionsEvent, callback);
|
||||
|
||||
const denied = missing.filter(
|
||||
@@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise<void> {
|
||||
);
|
||||
|
||||
if (denied.length > 0) {
|
||||
reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`));
|
||||
reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
@@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise<void> {
|
||||
androidx.core.app.ActivityCompat.requestPermissions(
|
||||
activity,
|
||||
toJavaStringArray(missing),
|
||||
PERMISSION_REQUEST_CODE,
|
||||
requestCode,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function ensureBlePermissions(): Promise<void> {
|
||||
await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth");
|
||||
}
|
||||
|
||||
export async function ensureVoicePermissions(): Promise<void> {
|
||||
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
|
||||
}
|
||||
@@ -1,19 +1,21 @@
|
||||
import { ImageSource } from "@nativescript/core";
|
||||
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
|
||||
import { loadDeviceAddresses } from "./device-addresses";
|
||||
import { ensureBlePermissions } from "./android-permissions";
|
||||
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
|
||||
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
|
||||
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { nightscoutBridge } from "../native/nightscout";
|
||||
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
|
||||
import { grayImageToPreviewSource } from "../native/gray-image-preview";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import {
|
||||
applyDashboardScreenTimeout,
|
||||
consumeDashboardTiledWakePaint,
|
||||
drawDashboard,
|
||||
getDashboardNightscoutSettings,
|
||||
getDashboardSystemCardName,
|
||||
isDashboardVoiceControlEnabled,
|
||||
receiveInput,
|
||||
resetDashboardSleepTimerAndWake,
|
||||
setDashboardBatteryLevels,
|
||||
@@ -21,6 +23,7 @@ import {
|
||||
setDashboardNightscoutApiToken,
|
||||
setDashboardNightscoutSiteUrl,
|
||||
setDashboardSystemCardName,
|
||||
setDashboardVoiceControlEnabled,
|
||||
} from "../ui/dashboard";
|
||||
|
||||
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
|
||||
@@ -118,6 +121,8 @@ class DashboardController {
|
||||
private offFrameMetrics: (() => void) | null = null;
|
||||
private offMedia: (() => void) | null = null;
|
||||
private offNightscout: (() => void) | null = null;
|
||||
private offVoiceStatus: (() => void) | null = null;
|
||||
private offVoiceWakeWord: (() => void) | null = null;
|
||||
private lastInput = "waiting...";
|
||||
private lastSys = "none yet";
|
||||
private renderInProgress = false;
|
||||
@@ -132,6 +137,7 @@ class DashboardController {
|
||||
startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(),
|
||||
startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(),
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -322,11 +328,20 @@ class DashboardController {
|
||||
});
|
||||
}
|
||||
});
|
||||
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.appendLog(state.status);
|
||||
});
|
||||
this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
|
||||
void this.handleWakeWord(keyword).catch((error) => {
|
||||
this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
|
||||
});
|
||||
});
|
||||
|
||||
await mediaControllerBridge.start();
|
||||
await nightscoutBridge.start();
|
||||
await communicator.start();
|
||||
await this.requestRender("initial");
|
||||
this.startVoiceControlIfEnabled();
|
||||
this.dashboardTimer = setInterval(() => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
@@ -360,8 +375,13 @@ class DashboardController {
|
||||
this.offMedia = null;
|
||||
this.offNightscout?.();
|
||||
this.offNightscout = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
if (communicator) {
|
||||
await communicator.close().catch(() => {});
|
||||
}
|
||||
@@ -397,6 +417,10 @@ class DashboardController {
|
||||
this.offMedia = null;
|
||||
this.offNightscout?.();
|
||||
this.offNightscout = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
|
||||
const communicator = this.communicator;
|
||||
this.communicator = null;
|
||||
@@ -413,6 +437,7 @@ class DashboardController {
|
||||
}
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
await communicator?.close().catch(() => {});
|
||||
} finally {
|
||||
stopForegroundNotification();
|
||||
@@ -445,6 +470,30 @@ class DashboardController {
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
|
||||
if (enabled) {
|
||||
try {
|
||||
await ensureVoicePermissions();
|
||||
} catch (error) {
|
||||
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
setDashboardVoiceControlEnabled(enabled);
|
||||
this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
|
||||
if (enabled) {
|
||||
this.startVoiceControlIfEnabled();
|
||||
} else {
|
||||
voiceControlBridge.stop();
|
||||
}
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
await this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private endTextSettingEdit(): void {
|
||||
const finishedKind = this.activeTextSettingEditorKind;
|
||||
this.editingSystemCardName = false;
|
||||
@@ -580,6 +629,29 @@ class DashboardController {
|
||||
}
|
||||
}
|
||||
|
||||
private async handleWakeWord(keyword: string): Promise<void> {
|
||||
const normalized = keyword.trim();
|
||||
if (normalized.length === 0) return;
|
||||
this.appendLog(`wake-word detected: ${normalized}`);
|
||||
const changed = resetDashboardSleepTimerAndWake();
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
if (changed) {
|
||||
await this.requestRender("initial");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (changed) {
|
||||
const image = drawDashboard();
|
||||
this.setDisplayPreview(grayImageToPreviewSource(image));
|
||||
}
|
||||
}
|
||||
|
||||
private startVoiceControlIfEnabled(): void {
|
||||
if (!this.communicator) return;
|
||||
if (!isDashboardVoiceControlEnabled()) return;
|
||||
voiceControlBridge.start();
|
||||
}
|
||||
|
||||
private clearDashboardTimer(): void {
|
||||
if (this.dashboardTimer) {
|
||||
clearInterval(this.dashboardTimer);
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private started = false;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
listener({ status: this.status });
|
||||
return () => this.statusListeners.delete(listener);
|
||||
}
|
||||
|
||||
onWakeWord(listener: (keyword: string) => void): () => void {
|
||||
this.wakeWordListeners.add(listener);
|
||||
return () => this.wakeWordListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.started || !global.isAndroid) return;
|
||||
this.ensureController();
|
||||
this.started = true;
|
||||
this.controller?.start();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started || !global.isAndroid) return;
|
||||
this.started = false;
|
||||
this.controller?.stop();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
|
||||
private ensureController(): void {
|
||||
if (!global.isAndroid || this.controller) return;
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (!context) {
|
||||
throw new Error("Android application context unavailable");
|
||||
}
|
||||
this.controller = new com.faceclaw.app.FaceclawVoiceController(context);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({
|
||||
onStatus: (status: string) => {
|
||||
this.setStatus(String(status));
|
||||
},
|
||||
onWakeWord: (keyword: string) => {
|
||||
for (const listener of this.wakeWordListeners) {
|
||||
listener(String(keyword));
|
||||
}
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private setStatus(status: string): void {
|
||||
this.status = status;
|
||||
for (const listener of this.statusListeners) {
|
||||
listener({ status });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
@@ -21,9 +21,11 @@ export type WakeModeSetting = "tiled" | "synchronized";
|
||||
export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw";
|
||||
export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s";
|
||||
export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled";
|
||||
export const DEFAULT_VOICE_CONTROL_ENABLED = false;
|
||||
const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName";
|
||||
const SCREEN_TIMEOUT_KEY = "display.screenTimeout";
|
||||
const WAKE_MODE_KEY = "display.wakeMode";
|
||||
const VOICE_CONTROL_ENABLED_KEY = "voice.enabled";
|
||||
const SYSTEM_CARD_SETTING_KEYS: Record<SystemCardSettingKey, string> = {
|
||||
showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo",
|
||||
showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators",
|
||||
@@ -138,6 +140,15 @@ export function saveWakeModeSetting(value: WakeModeSetting): WakeModeSetting {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function loadVoiceControlEnabled(): boolean {
|
||||
return ApplicationSettings.getBoolean(VOICE_CONTROL_ENABLED_KEY, DEFAULT_VOICE_CONTROL_ENABLED);
|
||||
}
|
||||
|
||||
export function saveVoiceControlEnabled(value: boolean): boolean {
|
||||
ApplicationSettings.setBoolean(VOICE_CONTROL_ENABLED_KEY, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting {
|
||||
switch (value) {
|
||||
case "15s":
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
loadScreenTimeoutSetting,
|
||||
loadSystemCardSettings,
|
||||
loadSystemCardName,
|
||||
loadVoiceControlEnabled,
|
||||
loadWakeModeSetting,
|
||||
nextScreenTimeoutSetting,
|
||||
nextWakeModeSetting,
|
||||
@@ -37,6 +38,7 @@ import {
|
||||
screenTimeoutMs,
|
||||
saveSystemCardSetting,
|
||||
saveSystemCardName,
|
||||
saveVoiceControlEnabled,
|
||||
wakeModeLabel,
|
||||
type DashboardSlotId,
|
||||
type NightscoutSettings,
|
||||
@@ -63,6 +65,7 @@ let dashboardState = {
|
||||
systemCardSettings: loadSystemCardSettings(),
|
||||
screenTimeout: loadScreenTimeoutSetting(),
|
||||
wakeMode: loadWakeModeSetting(),
|
||||
voiceControlEnabled: loadVoiceControlEnabled(),
|
||||
tiledWakePaintPending: false,
|
||||
nightscoutSettings: loadNightscoutSettings(),
|
||||
battery: {
|
||||
@@ -80,6 +83,7 @@ const dashboardActions: LayerActions = {
|
||||
startNightscoutSiteUrlEdit: () => {},
|
||||
startNightscoutApiTokenEdit: () => {},
|
||||
endTextSettingEdit: () => {},
|
||||
setVoiceControlEnabled: () => {},
|
||||
};
|
||||
const dashboardFont = loadEmbeddedTerminus12();
|
||||
const dashboardSystemFont = loadEmbeddedTerminus16();
|
||||
@@ -210,6 +214,14 @@ export function setDashboardSystemCardSetting(key: SystemCardSettingKey, value:
|
||||
saveSystemCardSetting(key, value);
|
||||
}
|
||||
|
||||
export function isDashboardVoiceControlEnabled(): boolean {
|
||||
return dashboardState.voiceControlEnabled;
|
||||
}
|
||||
|
||||
export function setDashboardVoiceControlEnabled(value: boolean): void {
|
||||
dashboardState.voiceControlEnabled = saveVoiceControlEnabled(value);
|
||||
}
|
||||
|
||||
export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean {
|
||||
const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout);
|
||||
if (timeoutMs === null || !dashboardState.screenOn) return false;
|
||||
@@ -511,6 +523,12 @@ function createSettingsMenuLayer(): MenuLayer {
|
||||
ctx.stack.push(createDashboardSettingsMenuLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Voice",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(createVoiceSettingsMenuLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Integrations",
|
||||
onSelect: (ctx) => {
|
||||
@@ -558,6 +576,24 @@ function setWakeMode(value: WakeModeSetting): void {
|
||||
dashboardState.wakeMode = saveWakeModeSetting(value);
|
||||
}
|
||||
|
||||
function createVoiceSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Voice",
|
||||
[
|
||||
{
|
||||
label: "Enable",
|
||||
onSelect: async (ctx) => {
|
||||
await ctx.actions.setVoiceControlEnabled(!dashboardState.voiceControlEnabled);
|
||||
},
|
||||
render: ({ image, x, y, width, selected }) => {
|
||||
drawToggleMenuItem(image, dashboardFont, x, y, width, "Enable", dashboardState.voiceControlEnabled, selected);
|
||||
},
|
||||
},
|
||||
],
|
||||
TOP_LEFT_MENU_LAYOUT,
|
||||
);
|
||||
}
|
||||
|
||||
function createDashboardSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Dashboard",
|
||||
|
||||
@@ -15,6 +15,7 @@ export type LayerActions = {
|
||||
startNightscoutSiteUrlEdit: () => Promise<void> | void;
|
||||
startNightscoutApiTokenEdit: () => Promise<void> | void;
|
||||
endTextSettingEdit: () => Promise<void> | void;
|
||||
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type PaintBelow = () => GrayImage;
|
||||
|
||||
Reference in new issue
Block a user