iOS wakeword support

This commit is contained in:
James Babcock committed 2026-09-19 01:37:36 -07:00
1 parent d1df4bf046
commit 15a3e129b2
17 files changed
+341 -137

No files matched your search

@@ -130,7 +130,7 @@ public class FaceclawVoiceController {
private String lastTranscript = "";
private volatile boolean saveRecordings;
private volatile boolean endpointing;
private final EndpointDetector endpointDetector = new EndpointDetector();
private final VoiceEndpointDetector endpointDetector = new VoiceEndpointDetector();
private java.io.ByteArrayOutputStream recordingPcm;
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
@@ -1028,111 +1028,6 @@ public class FaceclawVoiceController {
}
}
/**
* Decides when a hands-free utterance is over, so "Hey Even" capture can
* stop without a button release.
*
* Runs on the decoded 16 kHz PCM, so it works the same in every input mode
* (the cloud path never sees the samples on this side, and the onboard
* recognizer's own endpointing only covers ONBOARD).
*
* Timing is measured on the sample clock rather than the wall clock: BLE
* delivers mic packets in bursts, so elapsed real time badly overestimates
* how much audio has actually been heard.
*
* The threshold is relative to a noise floor measured over the first
* {@link #CALIBRATE_MS} of the session, which is roughly the interval where
* the user is reacting to the dialog appearing and not yet speaking.
*/
private static final class EndpointDetector {
/** Audio used to estimate the room's noise floor. */
private static final int CALIBRATE_MS = 300;
/** Speech must exceed this multiple of the noise floor to count as onset. */
private static final double ONSET_FACTOR = 3.0;
/** Below this multiple of the noise floor counts as silence again. */
private static final double RELEASE_FACTOR = 1.8;
/** Absolute floor, so a silent room can't make the threshold ~0. */
private static final double MIN_RMS = 220.0;
/** Trailing silence that ends an utterance. */
private static final int SILENCE_MS = 900;
/** If the user never speaks, give up rather than record forever. */
private static final int LEAD_IN_MS = 6000;
/** Hard cap on a single utterance. */
private static final int MAX_UTTERANCE_MS = 30000;
private long totalSamples;
private double noiseAccum;
private int noisePackets;
private double threshold;
private boolean speechStarted;
private long silenceSamples;
private boolean fired;
void reset() {
totalSamples = 0;
noiseAccum = 0;
noisePackets = 0;
threshold = 0;
speechStarted = false;
silenceSamples = 0;
fired = false;
}
/** Returns true exactly once, on the packet that ends the utterance. */
boolean accept(short[] pcm, int count) {
if (fired || count <= 0) {
return false;
}
totalSamples += count;
long elapsedMs = totalSamples * 1000L / SAMPLE_RATE;
double sumSquares = 0;
for (int i = 0; i < count; i++) {
double s = pcm[i];
sumSquares += s * s;
}
double rms = Math.sqrt(sumSquares / count);
if (elapsedMs <= CALIBRATE_MS) {
noiseAccum += rms;
noisePackets++;
return false;
}
if (threshold == 0) {
double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0;
threshold = Math.max(noiseFloor, MIN_RMS);
}
if (!speechStarted) {
if (rms >= threshold * ONSET_FACTOR) {
speechStarted = true;
silenceSamples = 0;
} else if (elapsedMs >= LEAD_IN_MS) {
// Never heard anything; close the dialog rather than hang.
fired = true;
return true;
}
return false;
}
if (rms < threshold * RELEASE_FACTOR) {
silenceSamples += count;
if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) {
fired = true;
return true;
}
} else {
silenceSamples = 0;
}
if (elapsedMs >= MAX_UTTERANCE_MS) {
fired = true;
return true;
}
return false;
}
}
private void stopG2Audio() {
FaceclawBleCommunicator currentCommunicator = communicator;
if (currentCommunicator != null) {
+1 -1
View File
@@ -5,7 +5,7 @@
+ (NSInteger)authorizationStatus;
+ (void)requestAuthorization:(void (^)(NSInteger status))completion;
/** Returns an actionable error, or an empty string when capture starts. */
- (NSString *)start;
- (NSString *)startWithEndpointing:(BOOL)endpointing;
- (void)acceptPacket:(NSData *)packet;
- (void)finish;
- (void)cancel;
+9 -2
View File
@@ -1,6 +1,7 @@
#import "FaceclawSpeech.h"
#import "FaceclawLc3Decoder.h"
#import "FaceclawSpeechTranscript.h"
#import <FaceclawKit/FaceclawKit.h>
#import <Speech/Speech.h>
#import <AVFoundation/AVFoundation.h>
#import <math.h>
@@ -16,6 +17,7 @@
@property(nonatomic) BOOL accepting;
@property(nonatomic, copy) NSString *bestText;
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
@end
@implementation FaceclawSpeech
- (instancetype)init {
@@ -33,7 +35,7 @@
NSData *json = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil];
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
}
- (NSString *)start {
- (NSString *)startWithEndpointing:(BOOL)endpointing {
[self cancel];
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
@@ -52,6 +54,7 @@
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
self.bestText = @""; self.accepting = YES;
self.transcript = [FaceclawSpeechTranscript new];
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
NSUInteger generation = self.generation;
__weak FaceclawSpeech *weakSelf = self;
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
@@ -89,11 +92,13 @@
NSUInteger generation = self.generation;
FaceclawLc3Decoder *decoder = self.decoder;
SFSpeechAudioBufferRecognitionRequest *request = self.request;
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
NSData *copy = [packet copy];
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
if (self.generation != generation) return;
NSData *pcm = [decoder decodePacket:copy];
double rms = 0;
BOOL speechEnded = NO;
if (pcm.length) {
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
@@ -105,11 +110,13 @@
}
rms = sqrt(rms / buffer.frameLength);
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
}
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
dispatch_async(dispatch_get_main_queue(), ^{
if (self.generation != generation) return;
self.pendingPackets--;
if (speechEnded && self.accepting) [self finish];
if (pcm.length && packets % 20 == 0) [self emit:@{@"kind":@"audio", @"packets":@(packets), @"seconds":@(packets * 0.05), @"rms":@(rms), @"missing":@(missing), @"errors":@(errors)}];
if (errors > 10 && !packets) [self complete:@"Could not decode glasses audio. Reconnect the glasses and retry."];
});
@@ -140,6 +147,6 @@
self.generation++;
self.accepting = NO; self.pendingPackets = 0;
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
self.bestText = @""; self.transcript = nil;
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
}
@end
+1 -1
View File
@@ -52,7 +52,7 @@ export const readString = (data: Uint8Array, field: number): string => protocol.
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
function input(event: any): GlassesInput | null {
return event ? { kind: event.kind === 'sys-event' && event.eventType === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : event.kind,
eventType: event.eventType, eventSource: event.eventSource, systemExitReasonCode: event.systemExitReasonCode,
+1 -1
View File
@@ -127,7 +127,7 @@ export function readString(data: Uint8Array, number: number): string { return Ar
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
const events = readBytes(message.payload, 13); if (!events) return null
+3 -3
View File
@@ -93,7 +93,7 @@ export class IosPreviewController {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
disconnect: () => this.disconnect(),
startVoiceCapture: () => this.startVoiceCapture(),
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
startTextSettingEdit: async setting => { await this.editSetting(setting) },
@@ -437,9 +437,9 @@ export class IosPreviewController {
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && this.session?.state.phase === 'connected'
}
private async startVoiceCapture(): Promise<void> {
private async startVoiceCapture(endpointing = false): Promise<void> {
if (!this.session || this.session.state.phase !== 'connected') return
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing)
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
+15 -5
View File
@@ -18,6 +18,8 @@ export class IosVoiceControlBridge {
private session: IosMicrophoneSession | null = null
private capturing = false
private finalizing = false
private completion: Promise<void> = Promise.resolve()
private resolveCompletion: (() => void) | null = null
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
@@ -48,12 +50,13 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
this.stop()
const generation = ++this.generation
this.log = log; this.session = session; this.packets = 0; this.capturing = true
const error = String(this.ensureNative().start() ?? '')
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
try {
await session.setMicrophone(true, packet => {
if (generation !== this.generation || !this.capturing) return
@@ -74,16 +77,19 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
stopPushToTalk(): void {
if (!this.capturing) return
stopPushToTalk(): Promise<void> {
const completion = this.completion
if (!this.capturing) return completion
this.capturing = false; this.finalizing = true
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus('Recognizing…'); this.native?.finish()
return completion
}
stop(): void {
++this.generation
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.finishCompletion()
}
handleSessionEnded(): void {
if (!this.capturing && !this.finalizing) return
@@ -95,6 +101,9 @@ export class IosVoiceControlBridge {
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
}
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
if (!this.capturing && !this.finalizing) return
if (event.kind === 'transcript') {
@@ -109,6 +118,7 @@ export class IosVoiceControlBridge {
} else if (event.kind === 'ended') {
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone()
this.finishCompletion()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
}
}
+5 -5
View File
@@ -130,10 +130,10 @@ function settingsSections(): SettingsSection[] {
// iOS offers cloud APIs; Android also supports local models and the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
// iOS has no local LLM, wakeword event delivery, or agent bridge yet.
...(!global.isIOS ? [localModelMenuItem()] : []),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
...(!global.isIOS ? [
localModelMenuItem(),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
textSettingMenuItem(assistantBridgeTokenSetting),
@@ -245,9 +245,9 @@ function settingsSections(): SettingsSection[] {
const deferred = new Set(["Navigate", "Watch"]);
return sections.map(section => {
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
if (section.label === "Voice") return { label: "Voice", items: [{
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
}] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
@@ -0,0 +1,66 @@
package com.faceclaw.app
import kotlin.math.max
import kotlin.math.sqrt
/** Ends a hands-free utterance using the 16 kHz sample clock, not BLE arrival timing.
* Shared by Android's voice controller and iOS's native audio queue.
*/
class VoiceEndpointDetector {
private var totalSamples = 0L
private var noiseAccum = 0.0
private var noisePackets = 0
private var threshold = 0.0
private var speechStarted = false
private var silenceSamples = 0L
private var fired = false
fun reset() {
totalSamples = 0
noiseAccum = 0.0
noisePackets = 0
threshold = 0.0
speechStarted = false
silenceSamples = 0
fired = false
}
fun accept(pcm: ShortArray, count: Int): Boolean {
if (fired || count <= 0) return false
require(count <= pcm.size)
var squares = 0.0
for (i in 0 until count) {
val sample = pcm[i].toDouble()
squares += sample * sample
}
return acceptLevel(sqrt(squares / count), count)
}
/** RMS in PCM16 units. Allows native decoders to reuse their existing level calculation. */
fun acceptLevel(rms: Double, sampleCount: Int): Boolean {
if (fired || sampleCount <= 0) return false
totalSamples += sampleCount
val elapsedMs = totalSamples * 1000 / 16000
if (elapsedMs <= 300) {
noiseAccum += rms
noisePackets++
return false
}
if (threshold == 0.0) threshold = max(if (noisePackets > 0) noiseAccum / noisePackets else 0.0, 220.0)
if (!speechStarted) {
if (rms >= threshold * 3.0) {
speechStarted = true
silenceSamples = 0
} else if (elapsedMs >= 6000) {
fired = true
}
return fired
}
if (rms < threshold * 1.8) {
silenceSamples += sampleCount
if (silenceSamples * 1000 / 16000 >= 900) fired = true
} else silenceSamples = 0
if (elapsedMs >= 30000) fired = true
return fired
}
}
@@ -128,7 +128,7 @@ class IosProtocol {
BleProtocol.readStringFieldValue(data.byteArray(), field)
fun glassesInput(data: NSData, sid: Int, flag: Int): G2Event? =
if (sid == BleProtocol.SID_EVENHUB && flag in intArrayOf(1, 6))
if (sid in intArrayOf(BleProtocol.SID_EVENHUB, BleProtocol.SID_EVEN_AI) && flag in intArrayOf(1, 6))
G2Event.decodePayload(sid, data.byteArray())
else null
+35 -6
View File
@@ -228,7 +228,7 @@ test('empty capture, rejection and closing never submit stale speech', async ()
}
});
function shellEnv() {
function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) {
const gestures = load('app/ui/gestures.ts', {});
class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } }
const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 };
@@ -244,11 +244,17 @@ function shellEnv() {
paint(_ctx, below) { return below(); }
handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); }
}
const settings = { wakeWordActionSetting: { get: () => 'voice-input' }, brightnessSetting: { get: () => 'auto' } };
const { shell } = load('app/ui/shell/shell.ts', {
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': {},
const voiceDialogs = [];
const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' },
assistantSkipConfirmationSetting: { get: () => skipConfirmation } };
const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', {
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}),
'../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu },
'./voice-input': {}, './keyboard-input': {}, './voice-activity': {}, './assistant': {},
'./voice-input': { VoiceInputLayer: class {
constructor(options) { this.options = options; voiceDialogs.push(options); }
startCapture() {}
onRemoved() { this.options.onClosed(); }
} }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {},
'../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {},
'../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {},
'./chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {},
@@ -280,9 +286,32 @@ function shellEnv() {
});
shell.registerWindow(app.window);
shell.focusWindow('ai-chat');
return { shell, app, input, menus, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
}
test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => {
for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) {
const env = shellEnv({ wakeAction, skipConfirmation: true });
env.shell.isAssistantAvailable = () => true;
env.shell.sleep();
const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 });
assert.equal(event.type, 'wakeword');
await env.shell.receiveInput(event);
await new Promise(resolve => setImmediate(resolve));
assert.equal(env.shell.isScreenOn(), wakeAction !== 'off');
assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0);
if (wakeAction === 'voice-input') {
const options = env.voiceDialogs[0];
assert.equal(options.handsFree, true);
assert.equal(options.autoSend, true);
assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant');
await env.shell.receiveInput(event);
assert.equal(env.voiceDialogs.length, 1);
}
env.shell.stack.clearToBase();
}
});
test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => {
const env = shellEnv();
await env.shell.receiveInput(env.event('long-press'));
+67
View File
@@ -42,6 +42,73 @@ test('iOS settings prompts honor password input and report cancellation', async
assert.equal(prompts[0].inputType, 'password');
assert.equal(accepted, false); assert.deepEqual(writes, []);
});
test('iOS host forwards hands-free capture mode to the voice bridge', async () => {
const captures = [];
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input'
? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} });
const host = Object.create(IosPreviewController.prototype);
host.session = { state: { phase: 'connected' } };
await host.startVoiceCapture(true);
await host.startVoiceCapture();
assert.deepEqual(captures, [true, false]);
});
for (const isIOS of [true, false]) {
test(`${isIOS ? 'iOS' : 'Android'} Assistant settings expose supported controls and usable model choices`, () => {
const store = new Map();
let picker;
const models = load('app/assistant/models.ts', { global: { isIOS }, require: () => ({
LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false,
}) });
const settings = load('app/ui/dashboard-settings.ts', { global: { isIOS }, require: id => {
if (id.includes('settings-store')) return {
onSettingsStoreChanged() {},
getStringSetting: (key, fallback) => store.get(key) ?? fallback,
setStringSetting: (key, value) => store.set(key, value),
getBooleanSetting: (key, fallback) => store.get(key) ?? fallback,
setBooleanSetting: (key, value) => store.set(key, value),
};
if (id === '~/assistant/models') return models;
return { Layer: class {}, isLocalModelReady: () => false,
openModalMenu: (_ctx, _title, items) => { picker = items; } };
} });
const menus = load('app/ui/dashboard/settings-menus.ts', { global: { isIOS }, require: id => {
if (id === '../dashboard-settings') return settings;
if (id === './settings-panel') return { SettingsPanelLayer: class { constructor(sections) { this.sections = sections; } } };
return { LOCAL_MODEL: { sizeBytes: 1000 }, ASR_MODELS: { moonshine: {}, 'whisper-base-en': {} },
uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) };
} });
const sections = menus.createSettingsPanelLayer().sections;
const assistant = sections.find(section => section.label === 'Assistant');
assert.equal(assistant.items.some(item => item.disabled === true), false);
const ctx = { stack: { pop() {} }, actions: { requestRender() {} } };
const row = setting => assistant.items.find(item => item.label === setting.label);
row(settings.assistantBackendSetting).onSelect(ctx);
assert.deepEqual(Array.from(picker, item => item.label), isIOS ? ['Cloud API'] : ['On-phone', 'My own agent (bridge)']);
assert.equal(assistant.items.some(item => item.label === 'On-phone model'), !isIOS);
assert.equal(!!row(settings.assistantBridgeHostSetting), !isIOS);
assert.equal(!!row(settings.assistantAllowProactiveSetting), !isIOS);
row(settings.assistantModelSetting).onSelect(ctx);
assert.equal(picker.some(item => item.label.includes('Qwen')), !isIOS);
const terra = picker.find(item => item.label === 'Terra');
assert.equal(terra.disabled(), true);
const keys = sections.find(section => section.label === 'API Keys').items;
assert.ok(keys.some(item => item.label === settings.openAiApiKeySetting.label));
assert.ok(keys.some(item => item.label === settings.anthropicApiKeySetting.label));
settings.openAiApiKeySetting.set('fixture-key');
assert.equal(terra.disabled(), false);
terra.onSelect(ctx);
assert.equal(settings.assistantModelSetting.get(), 'terra');
assert.equal(models.resolveAssistantModel(settings.assistantModelSetting.get(), {
openai: settings.openAiApiKeySetting.get(), anthropic: '',
}).provider, 'openai');
row(settings.assistantSkipConfirmationSetting).onSelect(ctx);
assert.equal(settings.assistantSkipConfirmationSetting.get(), true);
assert.ok(sections.find(section => section.label === 'Voice').items.some(item => item.label === settings.wakeWordActionSetting.label));
});
}
test('iOS phone battery distinguishes unknown, charging, full and unplugged readings', () => {
const device = { batteryLevel: -1, batteryState: 0 };
const api = load('app/native/phone-battery.ts', { require: () => ({}), global: { isIOS: true },
+29
View File
@@ -0,0 +1,29 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function renderSource(svg) {
let source;
const { rasterizeSvg } = loader({}, { './ios-graphics': {
rasterizeSvg: (value, size) => { source = value; assert.equal(size, 32); return 'rendered'; },
} })('app/native/svg-rasterizer.ios.ts');
assert.equal(rasterizeSvg(svg, 32, 2), 'rendered');
return source;
}
test('EvenHub currentColor SVG fills are resolved before handing them to SVGKit', () => {
// Same document shape as the Snake and Weather store icons copied from the phone.
const source = renderSource('<svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor"><path d="M4 1h10v2h-10Z"/></svg>');
assert.match(source, /fill="#ffffff"/);
assert.doesNotMatch(source, /currentColor/i);
assert.equal((source.match(/fill=/g) || []).length, 1);
assert.match(source, /viewBox="0 0 24 24"/);
});
test('currentColor strokes/styles resolve without filling open Lucide shapes', () => {
const source = renderSource('<svg fill="none" stroke="currentColor"><path style="stroke:currentColor" d="M2 2L8 8"/></svg>');
assert.match(source, /fill="none"/);
assert.match(source, /stroke="#ffffff"/);
assert.match(source, /style="stroke:#ffffff"/);
assert.match(source, /stroke-linecap="round"/);
});
+21 -1
View File
@@ -4,7 +4,7 @@ const fs = require('node:fs');
const vm = require('node:vm');
const ts = require('typescript');
function harness() {
const native = { start: () => '', cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const audio = [], timers = new Map(), commands = [], text = [];
let authorization = 3, permissionRequests = 0, release;
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
@@ -61,3 +61,23 @@ test('missing microphone data ends the session and disables the mic with an acti
for (const fn of h.timers.values()) fn();
assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/);
});
test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => {
const h = harness(); let ends = 0, completed = false;
h.bridge.onSpeechEnd(() => ends++);
const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start;
assert.equal(h.native.endpointing, true);
h.event({ kind: 'transcript', text: 'Set a timer', final: false });
h.event({ kind: 'finishing' });
const completion = h.bridge.stopPushToTalk().then(() => { completed = true; });
await Promise.resolve();
assert.equal(completed, false); assert.equal(ends, 1);
assert.deepEqual(h.commands, [true, false]);
h.event({ kind: 'transcript', text: 'Set a timer for five minutes.', final: true });
h.event({ kind: 'ended' }); await completion;
assert.equal(completed, true);
assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.');
const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual;
assert.equal(h.native.endpointing, false);
const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled;
});
@@ -0,0 +1,42 @@
package com.faceclaw.app
import kotlin.test.*
class VoiceEndpointDetectorTest {
@Test fun silenceEndsOnlyAfterSpeechAndFiresOnce() {
val detector = VoiceEndpointDetector()
repeat(6) { assertFalse(detector.acceptLevel(100.0, 800)) }
repeat(10) { assertFalse(detector.accept(ShortArray(800) { 2000 }, 800)) }
repeat(17) { assertFalse(detector.acceptLevel(100.0, 800)) }
assertTrue(detector.acceptLevel(100.0, 800))
assertFalse(detector.acceptLevel(100.0, 800))
detector.reset()
assertFalse(detector.acceptLevel(100.0, 800))
}
@Test fun pausesResetWhenSpeechResumesAndContinuousSpeechIsBounded() {
val detector = VoiceEndpointDetector()
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertFalse(detector.acceptLevel(2000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertFalse(detector.acceptLevel(2000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertTrue(detector.acceptLevel(0.0, 800))
detector.reset()
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
repeat(593) { assertFalse(detector.acceptLevel(2000.0, 800)) }
assertTrue(detector.acceptLevel(2000.0, 800))
}
@Test fun quietRoomTimesOutOnSampleClockAndNoiseCalibrationIsRelative() {
val detector = VoiceEndpointDetector()
repeat(119) { assertFalse(detector.acceptLevel(0.0, 800)) }
assertTrue(detector.acceptLevel(0.0, 800))
detector.reset()
repeat(6) { assertFalse(detector.acceptLevel(1000.0, 800)) }
repeat(20) { assertFalse(detector.acceptLevel(2000.0, 800)) }
assertFalse(detector.acceptLevel(4000.0, 800))
repeat(17) { assertFalse(detector.acceptLevel(1000.0, 800)) }
assertTrue(detector.acceptLevel(1000.0, 800))
}
}
@@ -47,6 +47,22 @@ internal actual fun inflateRecords(records: List<ByteArray>): List<ByteArray> =
}
class IosProtocolTest {
@Test
fun wakewordPacketsReachTheSharedEventDecoder() {
val api = IosProtocol()
val ctrl = BleProtocol.encodeVarintField(1, 1) // EVEN_AI_WAKE_UP
val payload = (BleProtocol.encodeVarintField(1, BleProtocol.EVEN_AI_CMD_CTRL) +
BleProtocol.encodeBytesField(3, ctrl)).data()
for (flag in listOf(1, 6)) {
val event = assertNotNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, flag))
assertEquals("even-ai", event.kind)
assertEquals(1, event.eventType)
}
assertNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, 2))
assertNull(api.glassesInput(payload, 0xff, 1))
assertNull(api.glassesInput(byteArrayOf().data(), BleProtocol.SID_EVEN_AI, 1))
}
@Test
fun platformClockAndWriteModes() {
val before = IosProtocolPlatform.elapsedRealtimeMs()
+28 -5
View File
@@ -13,27 +13,29 @@ function load(file, deps, globals = {}) {
return exports;
}
function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
function dialog({ autoSend = false, deferred = true, synchronousFinal, isIOS = false, handsFree = false } = {}) {
const listeners = new Set(), timers = new Map(), sent = [], refinements = [];
let complete, timerId = 0, starts = 0, frame;
let complete, timerId = 0, starts = 0, frame, speechEnd, endpointing;
let completion = new Promise(resolve => { complete = resolve; });
const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', {
'../../native/voice-control': { voiceControlBridge: {
onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); },
onStatus() { return () => {}; },
onSpeechEnd(cb) { speechEnd = cb; return () => { speechEnd = null; }; },
stop() {},
} },
'../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } },
'../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } },
'../gestures': { gestureHints: () => '' },
'./input-dialog': { paintInputDialog(_image, options) { frame = options; } },
}, {
global: { isIOS: false },
global: { isIOS },
setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; },
clearTimeout(id) { timers.delete(id); },
});
const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); };
const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(),
actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() {
const layer = new VoiceInputLayer({ autoSend, handsFree, onClosed() {}, dismiss: () => layer.onRemoved(),
actions: { requestRender() {}, startVoiceCapture(value) { starts++; endpointing = value; }, stopVoiceCapture() {
if (synchronousFinal) transcript(synchronousFinal, true);
return deferred ? completion : undefined;
} },
@@ -41,6 +43,7 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
});
layer.startCapture();
return { layer, sent, refinements, transcript, starts: () => starts,
endpoint: () => speechEnd?.(), endpointing: () => endpointing,
input: type => layer.handleInput({ type }, {}),
frame() { layer.paint({}, () => ({})); return frame; },
timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } },
@@ -50,6 +53,26 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
};
}
for (const autoSend of [true, false]) {
test(`iOS wakeword silence completion ${autoSend ? 'auto-sends' : 'asks for confirmation'} only after final recognition`, async () => {
const env = dialog({ isIOS: true, handsFree: true, autoSend });
assert.equal(env.endpointing(), true);
env.transcript('Set a timer');
env.endpoint(); env.timeout();
env.input('click'); // Cannot send a partial while native recognition drains.
assert.deepEqual(env.sent, []);
env.transcript('Set a timer for five minutes.', true);
await env.complete(); env.timeout();
if (!autoSend) {
assert.deepEqual(env.sent, []);
env.input('click');
}
assert.deepEqual(env.sent, ['Set a timer for five minutes.']);
env.endpoint(); env.timeout();
assert.equal(env.sent.length, 1);
});
}
for (const partial of ['', 'Set a timer']) {
test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => {
const env = dialog({ autoSend: true });