iOS wakeword support

This commit is contained in:
James Babcock committed 2026-09-19 01:37:36 -07:00
1 parent d1df4bf046
commit 15a3e129b2
17 files changed
+341 -137

No files matched your search

+35 -6
View File
@@ -228,7 +228,7 @@ test('empty capture, rejection and closing never submit stale speech', async ()
}
});
function shellEnv() {
function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) {
const gestures = load('app/ui/gestures.ts', {});
class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } }
const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 };
@@ -244,11 +244,17 @@ function shellEnv() {
paint(_ctx, below) { return below(); }
handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); }
}
const settings = { wakeWordActionSetting: { get: () => 'voice-input' }, brightnessSetting: { get: () => 'auto' } };
const { shell } = load('app/ui/shell/shell.ts', {
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': {},
const voiceDialogs = [];
const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' },
assistantSkipConfirmationSetting: { get: () => skipConfirmation } };
const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', {
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}),
'../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu },
'./voice-input': {}, './keyboard-input': {}, './voice-activity': {}, './assistant': {},
'./voice-input': { VoiceInputLayer: class {
constructor(options) { this.options = options; voiceDialogs.push(options); }
startCapture() {}
onRemoved() { this.options.onClosed(); }
} }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {},
'../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {},
'../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {},
'./chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {},
@@ -280,9 +286,32 @@ function shellEnv() {
});
shell.registerWindow(app.window);
shell.focusWindow('ai-chat');
return { shell, app, input, menus, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
}
test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => {
for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) {
const env = shellEnv({ wakeAction, skipConfirmation: true });
env.shell.isAssistantAvailable = () => true;
env.shell.sleep();
const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 });
assert.equal(event.type, 'wakeword');
await env.shell.receiveInput(event);
await new Promise(resolve => setImmediate(resolve));
assert.equal(env.shell.isScreenOn(), wakeAction !== 'off');
assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0);
if (wakeAction === 'voice-input') {
const options = env.voiceDialogs[0];
assert.equal(options.handsFree, true);
assert.equal(options.autoSend, true);
assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant');
await env.shell.receiveInput(event);
assert.equal(env.voiceDialogs.length, 1);
}
env.shell.stack.clearToBase();
}
});
test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => {
const env = shellEnv();
await env.shell.receiveInput(env.event('long-press'));
+67
View File
@@ -42,6 +42,73 @@ test('iOS settings prompts honor password input and report cancellation', async
assert.equal(prompts[0].inputType, 'password');
assert.equal(accepted, false); assert.deepEqual(writes, []);
});
test('iOS host forwards hands-free capture mode to the voice bridge', async () => {
const captures = [];
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input'
? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} });
const host = Object.create(IosPreviewController.prototype);
host.session = { state: { phase: 'connected' } };
await host.startVoiceCapture(true);
await host.startVoiceCapture();
assert.deepEqual(captures, [true, false]);
});
for (const isIOS of [true, false]) {
test(`${isIOS ? 'iOS' : 'Android'} Assistant settings expose supported controls and usable model choices`, () => {
const store = new Map();
let picker;
const models = load('app/assistant/models.ts', { global: { isIOS }, require: () => ({
LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false,
}) });
const settings = load('app/ui/dashboard-settings.ts', { global: { isIOS }, require: id => {
if (id.includes('settings-store')) return {
onSettingsStoreChanged() {},
getStringSetting: (key, fallback) => store.get(key) ?? fallback,
setStringSetting: (key, value) => store.set(key, value),
getBooleanSetting: (key, fallback) => store.get(key) ?? fallback,
setBooleanSetting: (key, value) => store.set(key, value),
};
if (id === '~/assistant/models') return models;
return { Layer: class {}, isLocalModelReady: () => false,
openModalMenu: (_ctx, _title, items) => { picker = items; } };
} });
const menus = load('app/ui/dashboard/settings-menus.ts', { global: { isIOS }, require: id => {
if (id === '../dashboard-settings') return settings;
if (id === './settings-panel') return { SettingsPanelLayer: class { constructor(sections) { this.sections = sections; } } };
return { LOCAL_MODEL: { sizeBytes: 1000 }, ASR_MODELS: { moonshine: {}, 'whisper-base-en': {} },
uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) };
} });
const sections = menus.createSettingsPanelLayer().sections;
const assistant = sections.find(section => section.label === 'Assistant');
assert.equal(assistant.items.some(item => item.disabled === true), false);
const ctx = { stack: { pop() {} }, actions: { requestRender() {} } };
const row = setting => assistant.items.find(item => item.label === setting.label);
row(settings.assistantBackendSetting).onSelect(ctx);
assert.deepEqual(Array.from(picker, item => item.label), isIOS ? ['Cloud API'] : ['On-phone', 'My own agent (bridge)']);
assert.equal(assistant.items.some(item => item.label === 'On-phone model'), !isIOS);
assert.equal(!!row(settings.assistantBridgeHostSetting), !isIOS);
assert.equal(!!row(settings.assistantAllowProactiveSetting), !isIOS);
row(settings.assistantModelSetting).onSelect(ctx);
assert.equal(picker.some(item => item.label.includes('Qwen')), !isIOS);
const terra = picker.find(item => item.label === 'Terra');
assert.equal(terra.disabled(), true);
const keys = sections.find(section => section.label === 'API Keys').items;
assert.ok(keys.some(item => item.label === settings.openAiApiKeySetting.label));
assert.ok(keys.some(item => item.label === settings.anthropicApiKeySetting.label));
settings.openAiApiKeySetting.set('fixture-key');
assert.equal(terra.disabled(), false);
terra.onSelect(ctx);
assert.equal(settings.assistantModelSetting.get(), 'terra');
assert.equal(models.resolveAssistantModel(settings.assistantModelSetting.get(), {
openai: settings.openAiApiKeySetting.get(), anthropic: '',
}).provider, 'openai');
row(settings.assistantSkipConfirmationSetting).onSelect(ctx);
assert.equal(settings.assistantSkipConfirmationSetting.get(), true);
assert.ok(sections.find(section => section.label === 'Voice').items.some(item => item.label === settings.wakeWordActionSetting.label));
});
}
test('iOS phone battery distinguishes unknown, charging, full and unplugged readings', () => {
const device = { batteryLevel: -1, batteryState: 0 };
const api = load('app/native/phone-battery.ts', { require: () => ({}), global: { isIOS: true },
+29
View File
@@ -0,0 +1,29 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function renderSource(svg) {
let source;
const { rasterizeSvg } = loader({}, { './ios-graphics': {
rasterizeSvg: (value, size) => { source = value; assert.equal(size, 32); return 'rendered'; },
} })('app/native/svg-rasterizer.ios.ts');
assert.equal(rasterizeSvg(svg, 32, 2), 'rendered');
return source;
}
test('EvenHub currentColor SVG fills are resolved before handing them to SVGKit', () => {
// Same document shape as the Snake and Weather store icons copied from the phone.
const source = renderSource('<svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor"><path d="M4 1h10v2h-10Z"/></svg>');
assert.match(source, /fill="#ffffff"/);
assert.doesNotMatch(source, /currentColor/i);
assert.equal((source.match(/fill=/g) || []).length, 1);
assert.match(source, /viewBox="0 0 24 24"/);
});
test('currentColor strokes/styles resolve without filling open Lucide shapes', () => {
const source = renderSource('<svg fill="none" stroke="currentColor"><path style="stroke:currentColor" d="M2 2L8 8"/></svg>');
assert.match(source, /fill="none"/);
assert.match(source, /stroke="#ffffff"/);
assert.match(source, /style="stroke:#ffffff"/);
assert.match(source, /stroke-linecap="round"/);
});
+21 -1
View File
@@ -4,7 +4,7 @@ const fs = require('node:fs');
const vm = require('node:vm');
const ts = require('typescript');
function harness() {
const native = { start: () => '', cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const audio = [], timers = new Map(), commands = [], text = [];
let authorization = 3, permissionRequests = 0, release;
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
@@ -61,3 +61,23 @@ test('missing microphone data ends the session and disables the mic with an acti
for (const fn of h.timers.values()) fn();
assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/);
});
test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => {
const h = harness(); let ends = 0, completed = false;
h.bridge.onSpeechEnd(() => ends++);
const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start;
assert.equal(h.native.endpointing, true);
h.event({ kind: 'transcript', text: 'Set a timer', final: false });
h.event({ kind: 'finishing' });
const completion = h.bridge.stopPushToTalk().then(() => { completed = true; });
await Promise.resolve();
assert.equal(completed, false); assert.equal(ends, 1);
assert.deepEqual(h.commands, [true, false]);
h.event({ kind: 'transcript', text: 'Set a timer for five minutes.', final: true });
h.event({ kind: 'ended' }); await completion;
assert.equal(completed, true);
assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.');
const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual;
assert.equal(h.native.endpointing, false);
const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled;
});
@@ -0,0 +1,42 @@
package com.faceclaw.app
import kotlin.test.*
class VoiceEndpointDetectorTest {
@Test fun silenceEndsOnlyAfterSpeechAndFiresOnce() {
val detector = VoiceEndpointDetector()
repeat(6) { assertFalse(detector.acceptLevel(100.0, 800)) }
repeat(10) { assertFalse(detector.accept(ShortArray(800) { 2000 }, 800)) }
repeat(17) { assertFalse(detector.acceptLevel(100.0, 800)) }
assertTrue(detector.acceptLevel(100.0, 800))
assertFalse(detector.acceptLevel(100.0, 800))
detector.reset()
assertFalse(detector.acceptLevel(100.0, 800))
}
@Test fun pausesResetWhenSpeechResumesAndContinuousSpeechIsBounded() {
val detector = VoiceEndpointDetector()
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertFalse(detector.acceptLevel(2000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertFalse(detector.acceptLevel(2000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertTrue(detector.acceptLevel(0.0, 800))
detector.reset()
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
repeat(593) { assertFalse(detector.acceptLevel(2000.0, 800)) }
assertTrue(detector.acceptLevel(2000.0, 800))
}
@Test fun quietRoomTimesOutOnSampleClockAndNoiseCalibrationIsRelative() {
val detector = VoiceEndpointDetector()
repeat(119) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertTrue(detector.acceptLevel(0.0, 800))
detector.reset()
repeat(6) { assertFalse(detector.acceptLevel(1000.0, 800)) }
repeat(20) { assertFalse(detector.acceptLevel(2000.0, 800)) }
assertFalse(detector.acceptLevel(4000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(1000.0, 800)) }
assertTrue(detector.acceptLevel(1000.0, 800))
}
}
@@ -47,6 +47,22 @@ internal actual fun inflateRecords(records: List<ByteArray>): List<ByteArray> =
}
class IosProtocolTest {
@Test
fun wakewordPacketsReachTheSharedEventDecoder() {
val api = IosProtocol()
val ctrl = BleProtocol.encodeVarintField(1, 1) // EVEN_AI_WAKE_UP
val payload = (BleProtocol.encodeVarintField(1, BleProtocol.EVEN_AI_CMD_CTRL) +
BleProtocol.encodeBytesField(3, ctrl)).data()
for (flag in listOf(1, 6)) {
val event = assertNotNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, flag))
assertEquals("even-ai", event.kind)
assertEquals(1, event.eventType)
}
assertNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, 2))
assertNull(api.glassesInput(payload, 0xff, 1))
assertNull(api.glassesInput(byteArrayOf().data(), BleProtocol.SID_EVEN_AI, 1))
}
@Test
fun platformClockAndWriteModes() {
val before = IosProtocolPlatform.elapsedRealtimeMs()
+28 -5
View File
@@ -13,27 +13,29 @@ function load(file, deps, globals = {}) {
return exports;
}
function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
function dialog({ autoSend = false, deferred = true, synchronousFinal, isIOS = false, handsFree = false } = {}) {
const listeners = new Set(), timers = new Map(), sent = [], refinements = [];
let complete, timerId = 0, starts = 0, frame;
let complete, timerId = 0, starts = 0, frame, speechEnd, endpointing;
let completion = new Promise(resolve => { complete = resolve; });
const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', {
'../../native/voice-control': { voiceControlBridge: {
onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); },
onStatus() { return () => {}; },
onSpeechEnd(cb) { speechEnd = cb; return () => { speechEnd = null; }; },
stop() {},
} },
'../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } },
'../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } },
'../gestures': { gestureHints: () => '' },
'./input-dialog': { paintInputDialog(_image, options) { frame = options; } },
}, {
global: { isIOS: false },
global: { isIOS },
setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; },
clearTimeout(id) { timers.delete(id); },
});
const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); };
const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(),
actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() {
const layer = new VoiceInputLayer({ autoSend, handsFree, onClosed() {}, dismiss: () => layer.onRemoved(),
actions: { requestRender() {}, startVoiceCapture(value) { starts++; endpointing = value; }, stopVoiceCapture() {
if (synchronousFinal) transcript(synchronousFinal, true);
return deferred ? completion : undefined;
} },
@@ -41,6 +43,7 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
});
layer.startCapture();
return { layer, sent, refinements, transcript, starts: () => starts,
endpoint: () => speechEnd?.(), endpointing: () => endpointing,
input: type => layer.handleInput({ type }, {}),
frame() { layer.paint({}, () => ({})); return frame; },
timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } },
@@ -50,6 +53,26 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
};
}
for (const autoSend of [true, false]) {
test(`iOS wakeword silence completion ${autoSend ? 'auto-sends' : 'asks for confirmation'} only after final recognition`, async () => {
const env = dialog({ isIOS: true, handsFree: true, autoSend });
assert.equal(env.endpointing(), true);
env.transcript('Set a timer');
env.endpoint(); env.timeout();
env.input('click'); // Cannot send a partial while native recognition drains.
assert.deepEqual(env.sent, []);
env.transcript('Set a timer for five minutes.', true);
await env.complete(); env.timeout();
if (!autoSend) {
assert.deepEqual(env.sent, []);
env.input('click');
}
assert.deepEqual(env.sent, ['Set a timer for five minutes.']);
env.endpoint(); env.timeout();
assert.equal(env.sent.length, 1);
});
}
for (const partial of ['', 'Set a timer']) {
test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => {
const env = dialog({ autoSend: true });