iOS wakeword support
This commit is contained in:
1 parent
d1df4bf046
commit
15a3e129b2
17 files changed
+341
-137
No files matched your search
@@ -130,7 +130,7 @@ public class FaceclawVoiceController {
|
||||
private String lastTranscript = "";
|
||||
private volatile boolean saveRecordings;
|
||||
private volatile boolean endpointing;
|
||||
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||
private final VoiceEndpointDetector endpointDetector = new VoiceEndpointDetector();
|
||||
private java.io.ByteArrayOutputStream recordingPcm;
|
||||
// Speaker verification against the enrolled wearer voice-print ("my voice
|
||||
// only" command gating). Configured before start(); the utterance PCM is
|
||||
@@ -1028,111 +1028,6 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||
* stop without a button release.
|
||||
*
|
||||
* Runs on the decoded 16 kHz PCM, so it works the same in every input mode
|
||||
* (the cloud path never sees the samples on this side, and the onboard
|
||||
* recognizer's own endpointing only covers ONBOARD).
|
||||
*
|
||||
* Timing is measured on the sample clock rather than the wall clock: BLE
|
||||
* delivers mic packets in bursts, so elapsed real time badly overestimates
|
||||
* how much audio has actually been heard.
|
||||
*
|
||||
* The threshold is relative to a noise floor measured over the first
|
||||
* {@link #CALIBRATE_MS} of the session, which is roughly the interval where
|
||||
* the user is reacting to the dialog appearing and not yet speaking.
|
||||
*/
|
||||
private static final class EndpointDetector {
|
||||
/** Audio used to estimate the room's noise floor. */
|
||||
private static final int CALIBRATE_MS = 300;
|
||||
/** Speech must exceed this multiple of the noise floor to count as onset. */
|
||||
private static final double ONSET_FACTOR = 3.0;
|
||||
/** Below this multiple of the noise floor counts as silence again. */
|
||||
private static final double RELEASE_FACTOR = 1.8;
|
||||
/** Absolute floor, so a silent room can't make the threshold ~0. */
|
||||
private static final double MIN_RMS = 220.0;
|
||||
/** Trailing silence that ends an utterance. */
|
||||
private static final int SILENCE_MS = 900;
|
||||
/** If the user never speaks, give up rather than record forever. */
|
||||
private static final int LEAD_IN_MS = 6000;
|
||||
/** Hard cap on a single utterance. */
|
||||
private static final int MAX_UTTERANCE_MS = 30000;
|
||||
|
||||
private long totalSamples;
|
||||
private double noiseAccum;
|
||||
private int noisePackets;
|
||||
private double threshold;
|
||||
private boolean speechStarted;
|
||||
private long silenceSamples;
|
||||
private boolean fired;
|
||||
|
||||
void reset() {
|
||||
totalSamples = 0;
|
||||
noiseAccum = 0;
|
||||
noisePackets = 0;
|
||||
threshold = 0;
|
||||
speechStarted = false;
|
||||
silenceSamples = 0;
|
||||
fired = false;
|
||||
}
|
||||
|
||||
/** Returns true exactly once, on the packet that ends the utterance. */
|
||||
boolean accept(short[] pcm, int count) {
|
||||
if (fired || count <= 0) {
|
||||
return false;
|
||||
}
|
||||
totalSamples += count;
|
||||
long elapsedMs = totalSamples * 1000L / SAMPLE_RATE;
|
||||
|
||||
double sumSquares = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
double s = pcm[i];
|
||||
sumSquares += s * s;
|
||||
}
|
||||
double rms = Math.sqrt(sumSquares / count);
|
||||
|
||||
if (elapsedMs <= CALIBRATE_MS) {
|
||||
noiseAccum += rms;
|
||||
noisePackets++;
|
||||
return false;
|
||||
}
|
||||
if (threshold == 0) {
|
||||
double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0;
|
||||
threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
}
|
||||
|
||||
if (!speechStarted) {
|
||||
if (rms >= threshold * ONSET_FACTOR) {
|
||||
speechStarted = true;
|
||||
silenceSamples = 0;
|
||||
} else if (elapsedMs >= LEAD_IN_MS) {
|
||||
// Never heard anything; close the dialog rather than hang.
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rms < threshold * RELEASE_FACTOR) {
|
||||
silenceSamples += count;
|
||||
if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) {
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
silenceSamples = 0;
|
||||
}
|
||||
|
||||
if (elapsedMs >= MAX_UTTERANCE_MS) {
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void stopG2Audio() {
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
if (currentCommunicator != null) {
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
+ (NSInteger)authorizationStatus;
|
||||
+ (void)requestAuthorization:(void (^)(NSInteger status))completion;
|
||||
/** Returns an actionable error, or an empty string when capture starts. */
|
||||
- (NSString *)start;
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing;
|
||||
- (void)acceptPacket:(NSData *)packet;
|
||||
- (void)finish;
|
||||
- (void)cancel;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#import "FaceclawSpeech.h"
|
||||
#import "FaceclawLc3Decoder.h"
|
||||
#import "FaceclawSpeechTranscript.h"
|
||||
#import <FaceclawKit/FaceclawKit.h>
|
||||
#import <Speech/Speech.h>
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <math.h>
|
||||
@@ -16,6 +17,7 @@
|
||||
@property(nonatomic) BOOL accepting;
|
||||
@property(nonatomic, copy) NSString *bestText;
|
||||
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
|
||||
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
|
||||
@end
|
||||
@implementation FaceclawSpeech
|
||||
- (instancetype)init {
|
||||
@@ -33,7 +35,7 @@
|
||||
NSData *json = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil];
|
||||
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (NSString *)start {
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing {
|
||||
[self cancel];
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
@@ -52,6 +54,7 @@
|
||||
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.transcript = [FaceclawSpeechTranscript new];
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
|
||||
@@ -89,11 +92,13 @@
|
||||
NSUInteger generation = self.generation;
|
||||
FaceclawLc3Decoder *decoder = self.decoder;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
|
||||
NSData *copy = [packet copy];
|
||||
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
|
||||
if (self.generation != generation) return;
|
||||
NSData *pcm = [decoder decodePacket:copy];
|
||||
double rms = 0;
|
||||
BOOL speechEnded = NO;
|
||||
if (pcm.length) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
|
||||
@@ -105,11 +110,13 @@
|
||||
}
|
||||
rms = sqrt(rms / buffer.frameLength);
|
||||
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
|
||||
}
|
||||
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.generation != generation) return;
|
||||
self.pendingPackets--;
|
||||
if (speechEnded && self.accepting) [self finish];
|
||||
if (pcm.length && packets % 20 == 0) [self emit:@{@"kind":@"audio", @"packets":@(packets), @"seconds":@(packets * 0.05), @"rms":@(rms), @"missing":@(missing), @"errors":@(errors)}];
|
||||
if (errors > 10 && !packets) [self complete:@"Could not decode glasses audio. Reconnect the glasses and retry."];
|
||||
});
|
||||
@@ -140,6 +147,6 @@
|
||||
self.generation++;
|
||||
self.accepting = NO; self.pendingPackets = 0;
|
||||
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
|
||||
self.bestText = @""; self.transcript = nil;
|
||||
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
|
||||
}
|
||||
@end
|
||||
@@ -52,7 +52,7 @@ export const readString = (data: Uint8Array, field: number): string => protocol.
|
||||
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
|
||||
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
|
||||
}
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
function input(event: any): GlassesInput | null {
|
||||
return event ? { kind: event.kind === 'sys-event' && event.eventType === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : event.kind,
|
||||
eventType: event.eventType, eventSource: event.eventSource, systemExitReasonCode: event.systemExitReasonCode,
|
||||
|
||||
@@ -127,7 +127,7 @@ export function readString(data: Uint8Array, number: number): string { return Ar
|
||||
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
|
||||
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
|
||||
}
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
|
||||
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
|
||||
const events = readBytes(message.payload, 13); if (!events) return null
|
||||
|
||||
@@ -93,7 +93,7 @@ export class IosPreviewController {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
@@ -437,9 +437,9 @@ export class IosPreviewController {
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && this.session?.state.phase === 'connected'
|
||||
}
|
||||
private async startVoiceCapture(): Promise<void> {
|
||||
private async startVoiceCapture(endpointing = false): Promise<void> {
|
||||
if (!this.session || this.session.state.phase !== 'connected') return
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing)
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
@@ -18,6 +18,8 @@ export class IosVoiceControlBridge {
|
||||
private session: IosMicrophoneSession | null = null
|
||||
private capturing = false
|
||||
private finalizing = false
|
||||
private completion: Promise<void> = Promise.resolve()
|
||||
private resolveCompletion: (() => void) | null = null
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
@@ -48,12 +50,13 @@ export class IosVoiceControlBridge {
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.session = session; this.packets = 0; this.capturing = true
|
||||
const error = String(this.ensureNative().start() ?? '')
|
||||
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
@@ -74,16 +77,19 @@ export class IosVoiceControlBridge {
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
stopPushToTalk(): void {
|
||||
if (!this.capturing) return
|
||||
stopPushToTalk(): Promise<void> {
|
||||
const completion = this.completion
|
||||
if (!this.capturing) return completion
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus('Recognizing…'); this.native?.finish()
|
||||
return completion
|
||||
}
|
||||
stop(): void {
|
||||
++this.generation
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.finishCompletion()
|
||||
}
|
||||
handleSessionEnded(): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
@@ -95,6 +101,9 @@ export class IosVoiceControlBridge {
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
}
|
||||
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
if (event.kind === 'transcript') {
|
||||
@@ -109,6 +118,7 @@ export class IosVoiceControlBridge {
|
||||
} else if (event.kind === 'ended') {
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.finishCompletion()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +130,10 @@ function settingsSections(): SettingsSection[] {
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
// iOS has no local LLM, wakeword event delivery, or agent bridge yet.
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
textSettingMenuItem(assistantBridgeTokenSetting),
|
||||
@@ -245,9 +245,9 @@ function settingsSections(): SettingsSection[] {
|
||||
const deferred = new Set(["Navigate", "Watch"]);
|
||||
return sections.map(section => {
|
||||
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
if (section.label === "Voice") return { label: "Voice", items: [{
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
|
||||
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
|
||||
}] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.math.max
|
||||
import kotlin.math.sqrt
|
||||
|
||||
/** Ends a hands-free utterance using the 16 kHz sample clock, not BLE arrival timing.
|
||||
* Shared by Android's voice controller and iOS's native audio queue.
|
||||
*/
|
||||
class VoiceEndpointDetector {
|
||||
private var totalSamples = 0L
|
||||
private var noiseAccum = 0.0
|
||||
private var noisePackets = 0
|
||||
private var threshold = 0.0
|
||||
private var speechStarted = false
|
||||
private var silenceSamples = 0L
|
||||
private var fired = false
|
||||
|
||||
fun reset() {
|
||||
totalSamples = 0
|
||||
noiseAccum = 0.0
|
||||
noisePackets = 0
|
||||
threshold = 0.0
|
||||
speechStarted = false
|
||||
silenceSamples = 0
|
||||
fired = false
|
||||
}
|
||||
|
||||
fun accept(pcm: ShortArray, count: Int): Boolean {
|
||||
if (fired || count <= 0) return false
|
||||
require(count <= pcm.size)
|
||||
var squares = 0.0
|
||||
for (i in 0 until count) {
|
||||
val sample = pcm[i].toDouble()
|
||||
squares += sample * sample
|
||||
}
|
||||
return acceptLevel(sqrt(squares / count), count)
|
||||
}
|
||||
|
||||
/** RMS in PCM16 units. Allows native decoders to reuse their existing level calculation. */
|
||||
fun acceptLevel(rms: Double, sampleCount: Int): Boolean {
|
||||
if (fired || sampleCount <= 0) return false
|
||||
totalSamples += sampleCount
|
||||
val elapsedMs = totalSamples * 1000 / 16000
|
||||
if (elapsedMs <= 300) {
|
||||
noiseAccum += rms
|
||||
noisePackets++
|
||||
return false
|
||||
}
|
||||
if (threshold == 0.0) threshold = max(if (noisePackets > 0) noiseAccum / noisePackets else 0.0, 220.0)
|
||||
if (!speechStarted) {
|
||||
if (rms >= threshold * 3.0) {
|
||||
speechStarted = true
|
||||
silenceSamples = 0
|
||||
} else if (elapsedMs >= 6000) {
|
||||
fired = true
|
||||
}
|
||||
return fired
|
||||
}
|
||||
if (rms < threshold * 1.8) {
|
||||
silenceSamples += sampleCount
|
||||
if (silenceSamples * 1000 / 16000 >= 900) fired = true
|
||||
} else silenceSamples = 0
|
||||
if (elapsedMs >= 30000) fired = true
|
||||
return fired
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ class IosProtocol {
|
||||
BleProtocol.readStringFieldValue(data.byteArray(), field)
|
||||
|
||||
fun glassesInput(data: NSData, sid: Int, flag: Int): G2Event? =
|
||||
if (sid == BleProtocol.SID_EVENHUB && flag in intArrayOf(1, 6))
|
||||
if (sid in intArrayOf(BleProtocol.SID_EVENHUB, BleProtocol.SID_EVEN_AI) && flag in intArrayOf(1, 6))
|
||||
G2Event.decodePayload(sid, data.byteArray())
|
||||
else null
|
||||
|
||||
|
||||
+35
-6
@@ -228,7 +228,7 @@ test('empty capture, rejection and closing never submit stale speech', async ()
|
||||
}
|
||||
});
|
||||
|
||||
function shellEnv() {
|
||||
function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) {
|
||||
const gestures = load('app/ui/gestures.ts', {});
|
||||
class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } }
|
||||
const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 };
|
||||
@@ -244,11 +244,17 @@ function shellEnv() {
|
||||
paint(_ctx, below) { return below(); }
|
||||
handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); }
|
||||
}
|
||||
const settings = { wakeWordActionSetting: { get: () => 'voice-input' }, brightnessSetting: { get: () => 'auto' } };
|
||||
const { shell } = load('app/ui/shell/shell.ts', {
|
||||
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': {},
|
||||
const voiceDialogs = [];
|
||||
const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' },
|
||||
assistantSkipConfirmationSetting: { get: () => skipConfirmation } };
|
||||
const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', {
|
||||
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}),
|
||||
'../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu },
|
||||
'./voice-input': {}, './keyboard-input': {}, './voice-activity': {}, './assistant': {},
|
||||
'./voice-input': { VoiceInputLayer: class {
|
||||
constructor(options) { this.options = options; voiceDialogs.push(options); }
|
||||
startCapture() {}
|
||||
onRemoved() { this.options.onClosed(); }
|
||||
} }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {},
|
||||
'../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {},
|
||||
'../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {},
|
||||
'./chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {},
|
||||
@@ -280,9 +286,32 @@ function shellEnv() {
|
||||
});
|
||||
shell.registerWindow(app.window);
|
||||
shell.focusWindow('ai-chat');
|
||||
return { shell, app, input, menus, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
|
||||
return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
|
||||
}
|
||||
|
||||
test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => {
|
||||
for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) {
|
||||
const env = shellEnv({ wakeAction, skipConfirmation: true });
|
||||
env.shell.isAssistantAvailable = () => true;
|
||||
env.shell.sleep();
|
||||
const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 });
|
||||
assert.equal(event.type, 'wakeword');
|
||||
await env.shell.receiveInput(event);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(env.shell.isScreenOn(), wakeAction !== 'off');
|
||||
assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0);
|
||||
if (wakeAction === 'voice-input') {
|
||||
const options = env.voiceDialogs[0];
|
||||
assert.equal(options.handsFree, true);
|
||||
assert.equal(options.autoSend, true);
|
||||
assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant');
|
||||
await env.shell.receiveInput(event);
|
||||
assert.equal(env.voiceDialogs.length, 1);
|
||||
}
|
||||
env.shell.stack.clearToBase();
|
||||
}
|
||||
});
|
||||
|
||||
test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => {
|
||||
const env = shellEnv();
|
||||
await env.shell.receiveInput(env.event('long-press'));
|
||||
|
||||
@@ -42,6 +42,73 @@ test('iOS settings prompts honor password input and report cancellation', async
|
||||
assert.equal(prompts[0].inputType, 'password');
|
||||
assert.equal(accepted, false); assert.deepEqual(writes, []);
|
||||
});
|
||||
|
||||
test('iOS host forwards hands-free capture mode to the voice bridge', async () => {
|
||||
const captures = [];
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input'
|
||||
? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} });
|
||||
const host = Object.create(IosPreviewController.prototype);
|
||||
host.session = { state: { phase: 'connected' } };
|
||||
await host.startVoiceCapture(true);
|
||||
await host.startVoiceCapture();
|
||||
assert.deepEqual(captures, [true, false]);
|
||||
});
|
||||
|
||||
for (const isIOS of [true, false]) {
|
||||
test(`${isIOS ? 'iOS' : 'Android'} Assistant settings expose supported controls and usable model choices`, () => {
|
||||
const store = new Map();
|
||||
let picker;
|
||||
const models = load('app/assistant/models.ts', { global: { isIOS }, require: () => ({
|
||||
LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false,
|
||||
}) });
|
||||
const settings = load('app/ui/dashboard-settings.ts', { global: { isIOS }, require: id => {
|
||||
if (id.includes('settings-store')) return {
|
||||
onSettingsStoreChanged() {},
|
||||
getStringSetting: (key, fallback) => store.get(key) ?? fallback,
|
||||
setStringSetting: (key, value) => store.set(key, value),
|
||||
getBooleanSetting: (key, fallback) => store.get(key) ?? fallback,
|
||||
setBooleanSetting: (key, value) => store.set(key, value),
|
||||
};
|
||||
if (id === '~/assistant/models') return models;
|
||||
return { Layer: class {}, isLocalModelReady: () => false,
|
||||
openModalMenu: (_ctx, _title, items) => { picker = items; } };
|
||||
} });
|
||||
const menus = load('app/ui/dashboard/settings-menus.ts', { global: { isIOS }, require: id => {
|
||||
if (id === '../dashboard-settings') return settings;
|
||||
if (id === './settings-panel') return { SettingsPanelLayer: class { constructor(sections) { this.sections = sections; } } };
|
||||
return { LOCAL_MODEL: { sizeBytes: 1000 }, ASR_MODELS: { moonshine: {}, 'whisper-base-en': {} },
|
||||
uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) };
|
||||
} });
|
||||
const sections = menus.createSettingsPanelLayer().sections;
|
||||
const assistant = sections.find(section => section.label === 'Assistant');
|
||||
assert.equal(assistant.items.some(item => item.disabled === true), false);
|
||||
const ctx = { stack: { pop() {} }, actions: { requestRender() {} } };
|
||||
const row = setting => assistant.items.find(item => item.label === setting.label);
|
||||
row(settings.assistantBackendSetting).onSelect(ctx);
|
||||
assert.deepEqual(Array.from(picker, item => item.label), isIOS ? ['Cloud API'] : ['On-phone', 'My own agent (bridge)']);
|
||||
assert.equal(assistant.items.some(item => item.label === 'On-phone model'), !isIOS);
|
||||
assert.equal(!!row(settings.assistantBridgeHostSetting), !isIOS);
|
||||
assert.equal(!!row(settings.assistantAllowProactiveSetting), !isIOS);
|
||||
row(settings.assistantModelSetting).onSelect(ctx);
|
||||
assert.equal(picker.some(item => item.label.includes('Qwen')), !isIOS);
|
||||
const terra = picker.find(item => item.label === 'Terra');
|
||||
assert.equal(terra.disabled(), true);
|
||||
const keys = sections.find(section => section.label === 'API Keys').items;
|
||||
assert.ok(keys.some(item => item.label === settings.openAiApiKeySetting.label));
|
||||
assert.ok(keys.some(item => item.label === settings.anthropicApiKeySetting.label));
|
||||
settings.openAiApiKeySetting.set('fixture-key');
|
||||
assert.equal(terra.disabled(), false);
|
||||
terra.onSelect(ctx);
|
||||
assert.equal(settings.assistantModelSetting.get(), 'terra');
|
||||
assert.equal(models.resolveAssistantModel(settings.assistantModelSetting.get(), {
|
||||
openai: settings.openAiApiKeySetting.get(), anthropic: '',
|
||||
}).provider, 'openai');
|
||||
row(settings.assistantSkipConfirmationSetting).onSelect(ctx);
|
||||
assert.equal(settings.assistantSkipConfirmationSetting.get(), true);
|
||||
assert.ok(sections.find(section => section.label === 'Voice').items.some(item => item.label === settings.wakeWordActionSetting.label));
|
||||
});
|
||||
}
|
||||
|
||||
test('iOS phone battery distinguishes unknown, charging, full and unplugged readings', () => {
|
||||
const device = { batteryLevel: -1, batteryState: 0 };
|
||||
const api = load('app/native/phone-battery.ts', { require: () => ({}), global: { isIOS: true },
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
function renderSource(svg) {
|
||||
let source;
|
||||
const { rasterizeSvg } = loader({}, { './ios-graphics': {
|
||||
rasterizeSvg: (value, size) => { source = value; assert.equal(size, 32); return 'rendered'; },
|
||||
} })('app/native/svg-rasterizer.ios.ts');
|
||||
assert.equal(rasterizeSvg(svg, 32, 2), 'rendered');
|
||||
return source;
|
||||
}
|
||||
|
||||
test('EvenHub currentColor SVG fills are resolved before handing them to SVGKit', () => {
|
||||
// Same document shape as the Snake and Weather store icons copied from the phone.
|
||||
const source = renderSource('<svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor"><path d="M4 1h10v2h-10Z"/></svg>');
|
||||
assert.match(source, /fill="#ffffff"/);
|
||||
assert.doesNotMatch(source, /currentColor/i);
|
||||
assert.equal((source.match(/fill=/g) || []).length, 1);
|
||||
assert.match(source, /viewBox="0 0 24 24"/);
|
||||
});
|
||||
|
||||
test('currentColor strokes/styles resolve without filling open Lucide shapes', () => {
|
||||
const source = renderSource('<svg fill="none" stroke="currentColor"><path style="stroke:currentColor" d="M2 2L8 8"/></svg>');
|
||||
assert.match(source, /fill="none"/);
|
||||
assert.match(source, /stroke="#ffffff"/);
|
||||
assert.match(source, /style="stroke:#ffffff"/);
|
||||
assert.match(source, /stroke-linecap="round"/);
|
||||
});
|
||||
@@ -4,7 +4,7 @@ const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
function harness() {
|
||||
const native = { start: () => '', cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [];
|
||||
let authorization = 3, permissionRequests = 0, release;
|
||||
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
|
||||
@@ -61,3 +61,23 @@ test('missing microphone data ends the session and disables the mic with an acti
|
||||
for (const fn of h.timers.values()) fn();
|
||||
assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/);
|
||||
});
|
||||
|
||||
test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => {
|
||||
const h = harness(); let ends = 0, completed = false;
|
||||
h.bridge.onSpeechEnd(() => ends++);
|
||||
const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start;
|
||||
assert.equal(h.native.endpointing, true);
|
||||
h.event({ kind: 'transcript', text: 'Set a timer', final: false });
|
||||
h.event({ kind: 'finishing' });
|
||||
const completion = h.bridge.stopPushToTalk().then(() => { completed = true; });
|
||||
await Promise.resolve();
|
||||
assert.equal(completed, false); assert.equal(ends, 1);
|
||||
assert.deepEqual(h.commands, [true, false]);
|
||||
h.event({ kind: 'transcript', text: 'Set a timer for five minutes.', final: true });
|
||||
h.event({ kind: 'ended' }); await completion;
|
||||
assert.equal(completed, true);
|
||||
assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.');
|
||||
const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual;
|
||||
assert.equal(h.native.endpointing, false);
|
||||
const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled;
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.*
|
||||
|
||||
class VoiceEndpointDetectorTest {
|
||||
@Test fun silenceEndsOnlyAfterSpeechAndFiresOnce() {
|
||||
val detector = VoiceEndpointDetector()
|
||||
repeat(6) { assertFalse(detector.acceptLevel(100.0, 800)) }
|
||||
repeat(10) { assertFalse(detector.accept(ShortArray(800) { 2000 }, 800)) }
|
||||
repeat(17) { assertFalse(detector.acceptLevel(100.0, 800)) }
|
||||
assertTrue(detector.acceptLevel(100.0, 800))
|
||||
assertFalse(detector.acceptLevel(100.0, 800))
|
||||
detector.reset()
|
||||
assertFalse(detector.acceptLevel(100.0, 800))
|
||||
}
|
||||
|
||||
@Test fun pausesResetWhenSpeechResumesAndContinuousSpeechIsBounded() {
|
||||
val detector = VoiceEndpointDetector()
|
||||
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
|
||||
assertFalse(detector.acceptLevel(2000.0, 800))
|
||||
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
|
||||
assertFalse(detector.acceptLevel(2000.0, 800))
|
||||
repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) }
|
||||
assertTrue(detector.acceptLevel(0.0, 800))
|
||||
detector.reset()
|
||||
repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) }
|
||||
repeat(593) { assertFalse(detector.acceptLevel(2000.0, 800)) }
|
||||
assertTrue(detector.acceptLevel(2000.0, 800))
|
||||
}
|
||||
|
||||
@Test fun quietRoomTimesOutOnSampleClockAndNoiseCalibrationIsRelative() {
|
||||
val detector = VoiceEndpointDetector()
|
||||
repeat(119) { assertFalse(detector.acceptLevel(0.0, 800)) }
|
||||
assertTrue(detector.acceptLevel(0.0, 800))
|
||||
detector.reset()
|
||||
repeat(6) { assertFalse(detector.acceptLevel(1000.0, 800)) }
|
||||
repeat(20) { assertFalse(detector.acceptLevel(2000.0, 800)) }
|
||||
assertFalse(detector.acceptLevel(4000.0, 800))
|
||||
repeat(17) { assertFalse(detector.acceptLevel(1000.0, 800)) }
|
||||
assertTrue(detector.acceptLevel(1000.0, 800))
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,22 @@ internal actual fun inflateRecords(records: List<ByteArray>): List<ByteArray> =
|
||||
}
|
||||
|
||||
class IosProtocolTest {
|
||||
@Test
|
||||
fun wakewordPacketsReachTheSharedEventDecoder() {
|
||||
val api = IosProtocol()
|
||||
val ctrl = BleProtocol.encodeVarintField(1, 1) // EVEN_AI_WAKE_UP
|
||||
val payload = (BleProtocol.encodeVarintField(1, BleProtocol.EVEN_AI_CMD_CTRL) +
|
||||
BleProtocol.encodeBytesField(3, ctrl)).data()
|
||||
for (flag in listOf(1, 6)) {
|
||||
val event = assertNotNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, flag))
|
||||
assertEquals("even-ai", event.kind)
|
||||
assertEquals(1, event.eventType)
|
||||
}
|
||||
assertNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, 2))
|
||||
assertNull(api.glassesInput(payload, 0xff, 1))
|
||||
assertNull(api.glassesInput(byteArrayOf().data(), BleProtocol.SID_EVEN_AI, 1))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun platformClockAndWriteModes() {
|
||||
val before = IosProtocolPlatform.elapsedRealtimeMs()
|
||||
|
||||
@@ -13,27 +13,29 @@ function load(file, deps, globals = {}) {
|
||||
return exports;
|
||||
}
|
||||
|
||||
function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
|
||||
function dialog({ autoSend = false, deferred = true, synchronousFinal, isIOS = false, handsFree = false } = {}) {
|
||||
const listeners = new Set(), timers = new Map(), sent = [], refinements = [];
|
||||
let complete, timerId = 0, starts = 0, frame;
|
||||
let complete, timerId = 0, starts = 0, frame, speechEnd, endpointing;
|
||||
let completion = new Promise(resolve => { complete = resolve; });
|
||||
const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', {
|
||||
'../../native/voice-control': { voiceControlBridge: {
|
||||
onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); },
|
||||
onStatus() { return () => {}; },
|
||||
onSpeechEnd(cb) { speechEnd = cb; return () => { speechEnd = null; }; },
|
||||
stop() {},
|
||||
} },
|
||||
'../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } },
|
||||
'../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } },
|
||||
'../gestures': { gestureHints: () => '' },
|
||||
'./input-dialog': { paintInputDialog(_image, options) { frame = options; } },
|
||||
}, {
|
||||
global: { isIOS: false },
|
||||
global: { isIOS },
|
||||
setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
});
|
||||
const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); };
|
||||
const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(),
|
||||
actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() {
|
||||
const layer = new VoiceInputLayer({ autoSend, handsFree, onClosed() {}, dismiss: () => layer.onRemoved(),
|
||||
actions: { requestRender() {}, startVoiceCapture(value) { starts++; endpointing = value; }, stopVoiceCapture() {
|
||||
if (synchronousFinal) transcript(synchronousFinal, true);
|
||||
return deferred ? completion : undefined;
|
||||
} },
|
||||
@@ -41,6 +43,7 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
|
||||
});
|
||||
layer.startCapture();
|
||||
return { layer, sent, refinements, transcript, starts: () => starts,
|
||||
endpoint: () => speechEnd?.(), endpointing: () => endpointing,
|
||||
input: type => layer.handleInput({ type }, {}),
|
||||
frame() { layer.paint({}, () => ({})); return frame; },
|
||||
timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } },
|
||||
@@ -50,6 +53,26 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
for (const autoSend of [true, false]) {
|
||||
test(`iOS wakeword silence completion ${autoSend ? 'auto-sends' : 'asks for confirmation'} only after final recognition`, async () => {
|
||||
const env = dialog({ isIOS: true, handsFree: true, autoSend });
|
||||
assert.equal(env.endpointing(), true);
|
||||
env.transcript('Set a timer');
|
||||
env.endpoint(); env.timeout();
|
||||
env.input('click'); // Cannot send a partial while native recognition drains.
|
||||
assert.deepEqual(env.sent, []);
|
||||
env.transcript('Set a timer for five minutes.', true);
|
||||
await env.complete(); env.timeout();
|
||||
if (!autoSend) {
|
||||
assert.deepEqual(env.sent, []);
|
||||
env.input('click');
|
||||
}
|
||||
assert.deepEqual(env.sent, ['Set a timer for five minutes.']);
|
||||
env.endpoint(); env.timeout();
|
||||
assert.equal(env.sent.length, 1);
|
||||
});
|
||||
}
|
||||
|
||||
for (const partial of ['', 'Set a timer']) {
|
||||
test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
|
||||
Reference in new issue
Block a user