diff --git a/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/Contents.json b/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/Contents.json new file mode 100644 index 00000000..aae26278 --- /dev/null +++ b/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/Contents.json @@ -0,0 +1,13 @@ +{ + "images": [ + { + "idiom": "universal", + "filename": "ic_tab_keyboard@3x.png", + "scale": "3x" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/ic_tab_keyboard@3x.png b/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/ic_tab_keyboard@3x.png new file mode 100644 index 00000000..f580af0e Binary files /dev/null and b/App_Resources/iOS/Assets.xcassets/ic_tab_keyboard.imageset/ic_tab_keyboard@3x.png differ diff --git a/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/Contents.json b/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/Contents.json new file mode 100644 index 00000000..b5dfb395 --- /dev/null +++ b/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/Contents.json @@ -0,0 +1,13 @@ +{ + "images": [ + { + "idiom": "universal", + "filename": "ic_tab_mic@3x.png", + "scale": "3x" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/ic_tab_mic@3x.png b/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/ic_tab_mic@3x.png new file mode 100644 index 00000000..6840cb66 Binary files /dev/null and b/App_Resources/iOS/Assets.xcassets/ic_tab_mic.imageset/ic_tab_mic@3x.png differ diff --git a/App_Resources/iOS/Info.plist b/App_Resources/iOS/Info.plist index 2d587079..0f165231 100644 --- a/App_Resources/iOS/Info.plist +++ b/App_Resources/iOS/Info.plist @@ -20,8 +20,10 @@ NSBluetoothAlwaysUsageDescription Faceclaw uses Bluetooth to connect to your glasses and ring, display your apps, and receive gestures. + NSMicrophoneUsageDescription + Faceclaw uses your phone microphone for voice input when your glasses are disconnected. NSSpeechRecognitionUsageDescription - Faceclaw turns audio from your glasses microphone into text for the app you choose. Dictation uses on-device recognition. + Faceclaw turns audio from your glasses or phone microphone into text for the app you choose. Dictation uses on-device recognition. NSCameraUsageDescription Faceclaw uses the camera to scan QR codes for app URLs you open in Developer. NSCalendarsUsageDescription diff --git a/App_Resources/iOS/src/FaceclawSpeech.h b/App_Resources/iOS/src/FaceclawSpeech.h index 49e14f8c..8f647292 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.h +++ b/App_Resources/iOS/src/FaceclawSpeech.h @@ -1,11 +1,14 @@ #import -/** On-device speech recognition fed ONLY by decoded G2 BLE microphone packets. */ +/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */ @interface FaceclawSpeech : NSObject @property(nonatomic, copy) void (^eventHandler)(NSString *json); + (NSInteger)authorizationStatus; ++ (NSInteger)microphoneAuthorizationStatus; ++ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion; + (void)requestAuthorization:(void (^)(NSInteger status))completion; /** Returns an actionable error, or an empty string when capture starts. */ - (NSString *)startWithEndpointing:(BOOL)endpointing; +- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing; - (void)acceptPacket:(NSData *)packet; - (void)finish; - (void)cancel; diff --git a/App_Resources/iOS/src/FaceclawSpeech.m b/App_Resources/iOS/src/FaceclawSpeech.m index 8da4da55..5487011e 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.m +++ b/App_Resources/iOS/src/FaceclawSpeech.m @@ -14,7 +14,11 @@ @property(nonatomic, strong) dispatch_queue_t audioQueue; @property(atomic) NSUInteger generation; @property(nonatomic) NSUInteger pendingPackets; -@property(nonatomic) BOOL accepting; +@property(atomic) BOOL accepting; +@property(nonatomic, strong) AVAudioEngine *phoneEngine; +@property(nonatomic) BOOL phoneTapInstalled; +@property(nonatomic) BOOL phoneSessionActive; +@property(nonatomic, strong) id interruptionObserver; @property(nonatomic, copy) NSString *bestText; @property(nonatomic, strong) FaceclawSpeechTranscript *transcript; @property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector; @@ -24,6 +28,12 @@ if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL); return self; } ++ (NSInteger)microphoneAuthorizationStatus { return [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]; } ++ (void)requestMicrophoneAuthorization:(void (^)(NSInteger))completion { + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { + dispatch_async(dispatch_get_main_queue(), ^{ completion(granted ? 3 : 2); }); + }]; +} + (NSInteger)authorizationStatus { return SFSpeechRecognizer.authorizationStatus; } + (void)requestAuthorization:(void (^)(NSInteger))completion { [SFSpeechRecognizer requestAuthorization:^(SFSpeechRecognizerAuthorizationStatus status) { @@ -36,7 +46,15 @@ if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]); } - (NSString *)startWithEndpointing:(BOOL)endpointing { + return [self startWithEndpointing:endpointing phone:NO]; +} +- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing { + return [self startWithEndpointing:endpointing phone:YES]; +} +- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone { [self cancel]; + if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized) + return @"Allow Microphone access for Faceclaw in iPhone Settings."; if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US"; @@ -44,8 +62,10 @@ if (!self.recognizer.supportsOnDeviceRecognition) return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language]; if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly."; - self.decoder = [FaceclawLc3Decoder new]; - if (!self.decoder) return @"Could not initialize the glasses audio decoder."; + if (!phone) { + self.decoder = [FaceclawLc3Decoder new]; + if (!self.decoder) return @"Could not initialize the glasses audio decoder."; + } self.recognizer.queue = NSOperationQueue.mainQueue; self.request = [SFSpeechAudioBufferRecognitionRequest new]; self.request.requiresOnDeviceRecognition = YES; @@ -77,7 +97,11 @@ } }); }]; - [self emit:@{@"kind":@"status", @"message":[NSString stringWithFormat:@"Listening on glasses (%@, on-device)…", language]}]; + if (phone) { + NSString *error = [self startPhoneAudio]; + if (error.length) { [self cancel]; return error; } + } + [self emit:@{@"kind":@"status", @"message":[NSString stringWithFormat:@"Listening on %@ (%@, on-device)…", phone ? @"phone" : @"glasses", language]}]; // Bound a dictation to the recognizer's supported short-session workflow. dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 55 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{ FaceclawSpeech *owner = weakSelf; @@ -85,6 +109,77 @@ }); return @""; } +- (NSString *)startPhoneAudio { + AVAudioSession *session = AVAudioSession.sharedInstance; + NSError *error = nil; + // Leave preferredInput unset so iOS selects its current default route, + // including a headset microphone when one is attached. + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord mode:AVAudioSessionModeMeasurement + options:AVAudioSessionCategoryOptionDefaultToSpeaker | AVAudioSessionCategoryOptionAllowBluetoothHFP error:&error]) + return [NSString stringWithFormat:@"Could not configure phone microphone: %@", error.localizedDescription]; + if (![session setActive:YES error:&error]) + return [NSString stringWithFormat:@"Could not activate phone microphone: %@", error.localizedDescription]; + self.phoneSessionActive = YES; + self.phoneEngine = [AVAudioEngine new]; + AVAudioInputNode *input = self.phoneEngine.inputNode; + AVAudioFormat *format = [input outputFormatForBus:0]; + if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again."; + NSUInteger generation = self.generation; + SFSpeechAudioBufferRecognitionRequest *request = self.request; + FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector; + __weak FaceclawSpeech *weakSelf = self; + [input installTapOnBus:0 bufferSize:1024 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { + FaceclawSpeech *owner = weakSelf; + if (!owner || owner.generation != generation || !owner.accepting || !buffer.frameLength) return; + // The tap owns buffer only for this callback. Copy before handing it + // to the serial recognition queue, which finish drains before endAudio. + AVAudioPCMBuffer *copy = [[AVAudioPCMBuffer alloc] initWithPCMFormat:buffer.format frameCapacity:buffer.frameLength]; + copy.frameLength = buffer.frameLength; + const AudioBufferList *source = buffer.audioBufferList; + AudioBufferList *destination = copy.mutableAudioBufferList; + for (UInt32 i = 0; i < source->mNumberBuffers; i++) + memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize); + dispatch_async(owner.audioQueue, ^{ @autoreleasepool { + if (owner.generation != generation) return; + [request appendAudioPCMBuffer:copy]; + double sum = 0; + if (copy.floatChannelData) { + const float *samples = copy.floatChannelData[0]; + NSUInteger stride = copy.format.interleaved ? copy.format.channelCount : 1; + for (NSUInteger i = 0; i < copy.frameLength; i++) sum += samples[i * stride] * samples[i * stride]; + } + double rms = sqrt(sum / copy.frameLength); + // The endpoint detector uses a 16 kHz sample clock; device input + // commonly arrives at 48 kHz. Recognition accepts its native rate. + int32_t samples = (int32_t)llround(copy.frameLength * 16000.0 / copy.format.sampleRate); + BOOL ended = [detector acceptLevelRms:rms * 32768.0 sampleCount:samples]; + if (ended) dispatch_async(dispatch_get_main_queue(), ^{ + if (owner.generation == generation && owner.accepting) [owner finish]; + }); + }}); + }]; + self.phoneTapInstalled = YES; + [self.phoneEngine prepare]; + if (![self.phoneEngine startAndReturnError:&error]) + return [NSString stringWithFormat:@"Could not start phone microphone: %@", error.localizedDescription]; + self.interruptionObserver = [NSNotificationCenter.defaultCenter addObserverForName:AVAudioSessionInterruptionNotification + object:session queue:NSOperationQueue.mainQueue usingBlock:^(NSNotification *notification) { + FaceclawSpeech *owner = weakSelf; + if (owner.generation == generation && [notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue] == AVAudioSessionInterruptionTypeBegan) + [owner complete:@"Phone microphone interrupted. Start voice input again to continue."]; + }]; + return @""; +} +- (void)stopPhoneAudio { + if (self.interruptionObserver) [NSNotificationCenter.defaultCenter removeObserver:self.interruptionObserver]; + self.interruptionObserver = nil; + [self.phoneEngine stop]; + if (self.phoneTapInstalled) [self.phoneEngine.inputNode removeTapOnBus:0]; + self.phoneTapInstalled = NO; + self.phoneEngine = nil; + if (self.phoneSessionActive) [AVAudioSession.sharedInstance setActive:NO withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation error:nil]; + self.phoneSessionActive = NO; +} - (void)acceptPacket:(NSData *)packet { if (!self.accepting) return; if (self.pendingPackets >= 40) { [self complete:@"Audio processing fell behind. Please try again."]; return; } @@ -125,6 +220,7 @@ - (void)finish { if (!self.accepting) return; self.accepting = NO; + [self stopPhoneAudio]; NSUInteger generation = self.generation; SFSpeechAudioBufferRecognitionRequest *request = self.request; [self emit:@{@"kind":@"finishing"}]; @@ -146,6 +242,7 @@ - (void)cancel { self.generation++; self.accepting = NO; self.pendingPackets = 0; + [self stopPhoneAudio]; [self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil; self.bestText = @""; self.transcript = nil; self.endpointDetector = nil; } diff --git a/CHANGELOG b/CHANGELOG index 26f499e9..b50ab386 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,7 @@ ## 0.7.3 - Future * Add iOS calendar integration +* iOS main screen matches the Android version's styling ## 0.7.2 - 2026-09-19 diff --git a/app/app.css b/app/app.css index d95f977e..49c6dbee 100644 --- a/app/app.css +++ b/app/app.css @@ -1,5 +1,6 @@ -@import '@nativescript/theme/css/core.css'; +@import '@nativescript/theme/css/core.css'; @import '@nativescript/theme/css/default.css'; +@import './phone-ui/remote-controls.css'; Page { background-color: #ffffff; @@ -171,139 +172,6 @@ Button.-primary { margin-bottom: 12; } -/* The phone's controller for the glasses (phone-ui/remote-controls.xml). */ -.remote-controls { - margin: 0 8 0 8; -} -/* The Settings / Watch / Ring tab bar above the controls. */ -.controls-tab-bar { - margin-bottom: 8; -} -.tab-button { - font-size: 14; - padding: 8 0; - margin: 0 2; - border-radius: 6; - background-color: #e2e2e2; - color: #333333; -} -.tab-button-selected { - background-color: #424242; - color: #ffffff; -} -/* The wakeword and keyboard buttons at the tab bar's right edge: icon-only, - * narrower than the tabs, and never in the selected state (they aren't tabs). - * Each is a button-styled GridLayout (Buttons can't hold images), so it - * borrows the .tab-button look and adds the elevation MaterialButtons get for - * free. */ -.tab-icon-button { - width: 56; - /* The real MaterialButtons render 48dp tall inside a 60dp layout box (6dp - * insets top and bottom); the margins and explicit height reproduce that. - * The height also bounds the tile's measure, without which its star row - * would collapse to content and pin the icon to the top. */ - height: 48; - margin: 6 2; - android-elevation: 2; -} -.tab-icon { - height: 20; - tint-color: #333333; -} -/* Landscape uses the full page height and a shared left edge for tabs/pad. */ -.landscape-layout .remote-controls { - margin: 0; -} -.landscape-layout .controls-tab-bar { - margin-bottom: 2; -} -.landscape-layout .tab-button { - margin-left: 1; - margin-right: 1; - min-width: 0; -} -.landscape-layout .tab-button[col="0"] { - margin-left: 0; -} -.landscape-layout .tab-icon-button { - width: 48; -} -.landscape-layout .tab-icon-button[col="4"] { - margin-right: 0; -} -.ns-dark .tab-icon { - tint-color: #cccccc; -} -.ns-dark .tab-button { - background-color: #333333; - color: #cccccc; -} -.ns-dark .tab-button-selected { - background-color: #e2e2e2; - color: #222222; -} -/* Settings tab rows. */ -.settings-row { - margin-bottom: 8; -} -.settings-row-label { - font-size: 14; - margin-right: 10; -} -.mode-button { - font-size: 13; - padding: 6 10; - margin-right: 6; -} -.brightness-icon { - font-size: 16; - margin: 0 4 0 4; - color: #444444; -} -.auto-toggle Label { - font-size: 13; - margin-right: 2; -} -.touchpad { - height: 230; - background-color: #1f1f1f; - border-radius: 14; - border-width: 1; - border-color: #3a3a3a; - color: #f0f0f0; - margin-top: 2; -} -.pad-focus { - font-size: 20; - color: #ffffff; -} -.pad-legend { - font-size: 11; - color: #8a8a8a; - margin-bottom: 8; -} -.action-row { - margin-top: 6; -} -.action-button { - margin: 2; - font-size: 14; -} -/* Ring tab: the same touchpad, but portrait like the ring's own pad. */ -.ring-touchpad { - width: 150; - height: 250; -} -/* Cosmetic marks mirroring the six rectangles on the physical ring's touch - strip; barely lighter than the pad so they read as texture, not buttons. */ -.ring-pad-marking { - width: 18; - height: 5; - border-radius: 2; - background-color: #2a2a2a; - horizontal-align: center; - vertical-align: center; -} /* * Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product * art (or a colourway swatch) on the left, decoded SKU + proximity + serial in diff --git a/app/app.ios.css b/app/app.ios.css index 318a0809..121a6bf7 100644 --- a/app/app.ios.css +++ b/app/app.ios.css @@ -1,4 +1,5 @@ @import '@nativescript/theme/css/core.css'; +@import './phone-ui/remote-controls.css'; .pairing-instructions { color: #555555; @@ -159,3 +160,47 @@ Button { margin-right: 4; font-size: 15; } + + +Page.main-page { + background-color: #ffffff; + color: #222222; +} +.main-page .main-header { + background-color: #424242; + color: #ffffff; + padding: 0 0 0 16; + height: 56; +} +.main-page .action-bar-title { font-size: 18; font-weight: bold; color: #ffffff; } +.main-page .action-bar-status { font-size: 14; color: #ffffff; margin-right: 12; } +.main-page .overflow-button { + background-color: transparent; + color: #ffffff; + font-size: 24; + width: 44; + height: 48; + margin: 0; + padding: 0; +} +/* Match Android's button layout boxes, including the inset around each tile. */ +.main-page Button.tab-button { height: 48; margin-top: 6; margin-bottom: 6; } +.main-page .mode-button { background-color: #e2e2e2; color: #333333; border-radius: 6; } +.main-page .ble-bandwidth-indicator { color: #777777; font-size: 12; padding: 2 8; } + +.main-page .system-name-modal { + background-color: #eeeeee; + color: #222222; + padding: 16; +} +.main-page .system-name-modal TextView { + margin: 0; + padding: 4; + background-color: #ffffff; + color: #222222; + placeholder-color: #666666; +} + +.main-landscape .system-name-modal { padding: 6 12; } +.main-landscape .system-name-modal .h2 { visibility: collapse; } +.main-landscape .system-name-modal Button { padding: 6 8; margin-top: 0; margin-bottom: 0; } diff --git a/app/g2/brightness-protocol.ts b/app/g2/brightness-protocol.ts new file mode 100644 index 00000000..79b64d57 --- /dev/null +++ b/app/g2/brightness-protocol.ts @@ -0,0 +1,8 @@ +import { bytes, concat, integer } from './ble-protocol' + +/** Same G2SettingPackage as Kotlin BleProtocol.buildSetBrightness. */ +export function setBrightness(magic: number, level: number | null): Uint8Array { + const fields = level === null ? integer(1, 1) + : concat(integer(1, 0), integer(2, Math.max(0, Math.min(100, Math.round(level))))) + return concat(integer(1, 1), integer(2, magic), bytes(3, bytes(1, fields))) +} diff --git a/app/g2/glasses-session.ts b/app/g2/glasses-session.ts index 1a10f9df..2152f327 100644 --- a/app/g2/glasses-session.ts +++ b/app/g2/glasses-session.ts @@ -1,3 +1,4 @@ +import { setBrightness } from './brightness-protocol' import { AncsClient, ANCS_FIRMWARE_VERSION } from './ancs-client' import { type CompassEvent } from '../native/compass-types' import * as protocol from './ble-protocol' @@ -423,6 +424,10 @@ export class GlassesSession { this.check(generation) await this.send('left', protocol.SID.settings, 0x20, queryWearState()) } + async setBrightness(level: number | null): Promise { + if (this.state.phase !== 'connected') return + await this.request('right', protocol.SID.settings, magic => setBrightness(magic, level), 'Brightness') + } setFrame(gray: Uint8Array): void { if (this.state.phase !== 'connected') return this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0) diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 95c5c0a5..7e4fac3e 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -1,3 +1,4 @@ +import type { KeyboardInputSession } from '../ui/shell/keyboard-input' import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup } from '../native/notification-icons.ios' import { shouldShowNotificationOnGlasses } from '../native/notification-sources' @@ -42,7 +43,7 @@ import { TextViewerLayer } from '../apps/files/text-viewer' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry' import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged, - previewColorSetting, lockScreenEnabledSetting } from '../ui/dashboard-settings' + previewColorSetting, lockScreenEnabledSetting, brightnessSetting, brightnessSettingToLevel } from '../ui/dashboard-settings' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state' import { findSoundEffect, playSoundEffect } from '../ui/sound-effects' @@ -83,13 +84,13 @@ export class IosPreviewController { private renderTimer: ReturnType | null = null private clockTimer: ReturnType | null = null private offSettings: (() => void) | null = null + private lastBrightness: string | null = null private active = false private runtimeRunning = false private clockMinute = -1 private shellDirty = true private inputQueue: Promise = Promise.resolve() private lastLayout = '' - private prompting = false private readonly appHosts = new Map() private readonly inProcessApps = new Map() private readonly batteryObservers: any[] = [] @@ -124,7 +125,8 @@ export class IosPreviewController { constructor(private readonly onFrame: (image: ImageSource, focus: string) => void, private readonly onError: (message: string) => void, - private readonly onConnectionState: (state: SessionState) => void = () => {}) { + private readonly onConnectionState: (state: SessionState) => void = () => {}, + private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}) { this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' }) this.compositor.configureSurface(LOCK_SCREEN_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: 1000, transparency: 'opaque', @@ -133,6 +135,13 @@ export class IosPreviewController { shell.configure({ actions: this.actions, voiceInputEnabled: true, + onKeyboardInputChanged: session => this.onKeyboardInputChanged(session ? { + targets: session.targets, + setText: text => { if (this.active && !this.glassesLocked) session.setText(text) }, + send: () => { if (this.active && !this.glassesLocked) session.send() }, + sendTo: id => { if (this.active && !this.glassesLocked) session.sendTo(id) }, + discard: () => session.discard(), + } : null), prepareVoiceCapture: () => this.prepareVoiceCapture(), getScreenTimeoutMs: () => null, requestShellRender: () => this.requestShellRender(), @@ -184,6 +193,7 @@ export class IosPreviewController { pause(): void { if (!this.active) return this.active = false + iosVoiceInput.stopPhoneCapture() // The phone preview is hidden; the glasses still own their screen and input. this.syncRuntime() this.logBluetooth(`Phone background; glasses ${this.session?.state.phase ?? 'disconnected'}`) @@ -230,12 +240,24 @@ export class IosPreviewController { this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { this.syncLockSetting() + this.pushBrightness() this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() }) this.clockTimer = setInterval(() => this.refreshClock(), 60_000) } + private pushBrightness(): void { + const session = this.session + if (session?.state.phase !== 'connected') { this.lastBrightness = null; return } + const value = brightnessSetting.get() + if (value === this.lastBrightness) return + this.lastBrightness = value + void session.setBrightness(brightnessSettingToLevel(value)).catch(error => { + this.lastBrightness = null + this.logBluetooth(`Brightness: ${String(error)}`) + }) + } private syncLockSetting(): void { const enabled = lockScreenEnabledSetting.get() if (enabled === this.lockEnabled) return @@ -500,6 +522,7 @@ export class IosPreviewController { if (!this.session) { this.session = new GlassesSession(iosBluetooth(), state => { if (state.phase !== "connected") resetRingInputFilter() + this.pushBrightness() this.maybePlayWelcomeSound(state) if (state.phase !== 'connected' && state.phase !== 'connecting') this.glassesWorn = null if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded() @@ -540,17 +563,22 @@ export class IosPreviewController { startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() } private async prepareVoiceCapture(): Promise { if (this.glassesLocked) return false - if (this.session?.state.phase !== 'connected') { - if (this.active) this.onError('Connect the glasses to use their microphone.') - return false - } - const ready = await iosVoiceInput.prepare(this.active) + const usePhoneMic = this.session?.state.phase !== 'connected' + if (usePhoneMic && !this.active) return false + const ready = await iosVoiceInput.prepare(this.active, usePhoneMic) if (!ready && this.active) this.onError(iosVoiceInput.statusText) - return ready && !this.glassesLocked && this.session?.state.phase === 'connected' + return ready && !this.glassesLocked && (this.session?.state.phase === 'connected' || this.active) } private async startVoiceCapture(endpointing = false): Promise { - if (this.glassesLocked || !this.session || this.session.state.phase !== 'connected') return - await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing) + if (this.glassesLocked) return + const log = (message: string) => this.logBluetooth(message) + if (this.session?.state.phase === 'connected') { + await iosVoiceInput.startGlassesCapture(this.session, log, endpointing) + } else if (this.active) { + // Recheck microphone permission if the glasses disconnected after prepare. + if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return + await iosVoiceInput.startPhoneCapture(log, endpointing) + } } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` @@ -568,17 +596,9 @@ export class IosPreviewController { try { File.fromPath(path.join(knownFolders.documents().path, 'bluetooth.log')).writeTextSync(this.logLines.join('\n')) } catch (error) { console.warn(`Bluetooth log: ${error}`) } } - async typeIntoApp(): Promise { - if (this.glassesLocked) return - if (!this.active) { this.logBluetooth('Text input requires opening Faceclaw on the phone'); return } - if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return - this.prompting = true - try { - const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title, - message: 'Enter text to send to this app.', okButtonText: 'Send', cancelButtonText: 'Cancel' }) - if (result.result && !this.glassesLocked) shell.sendTextToForegroundWindow(result.text) - } catch (error) { this.fail(error) } - finally { this.prompting = false } + typeIntoApp(): void { + if (this.glassesLocked || !this.active) return + shell.startKeyboardInput() } private async editSetting(setting: { editorTitle: string; inputKind?: string; get(): string; set(value: string): void }): Promise { if (!this.active) { this.logBluetooth('Editing text settings requires opening Faceclaw on the phone'); return false } diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index d6333bc3..a1095779 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -6,7 +6,7 @@ export type IosMicrophoneSession = { } /** iOS implementation of the shared voice dialog's event bridge. Audio comes - * exclusively from the connected glasses, never from the phone microphone. + * from the glasses when connected, or the default phone input in preview mode. */ export class IosVoiceControlBridge { private native: any = null @@ -16,6 +16,7 @@ export class IosVoiceControlBridge { private status = 'Voice input ready.' private generation = 0 private session: IosMicrophoneSession | null = null + private source: 'glasses' | 'phone' | null = null private capturing = false private finalizing = false private completion: Promise = Promise.resolve() @@ -40,20 +41,28 @@ export class IosVoiceControlBridge { } return this.native } - async prepare(foreground: boolean): Promise { + async prepare(foreground: boolean, usePhoneMic = false): Promise { let status = FaceclawSpeech.authorizationStatus() if (status === 0) { if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false } status = await new Promise(resolve => FaceclawSpeech.requestAuthorization(resolve)) } if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false } + if (usePhoneMic) { + let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus() + if (microphoneStatus === 0) { + if (!foreground) { this.setStatus('Open Faceclaw on the phone to allow microphone access.'); return false } + microphoneStatus = await new Promise(resolve => FaceclawSpeech.requestMicrophoneAuthorization(resolve)) + } + if (microphoneStatus !== 3) { this.setStatus('Allow Microphone access for Faceclaw in iPhone Settings.'); return false } + } return true } get statusText(): string { return this.status } async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise { this.stop() const generation = ++this.generation - this.log = log; this.session = session; this.packets = 0; this.capturing = true + this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '') if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } @@ -77,6 +86,18 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } + async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise { + this.stop() + this.log = log; this.source = 'phone'; this.capturing = true + this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) + const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '') + if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } + log('Voice: phone default microphone enabled; using on-device recognition') + } + stopPhoneCapture(): void { + if (this.source !== 'phone') return + this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd() + } stopPushToTalk(): Promise { const completion = this.completion if (!this.capturing) return completion @@ -88,11 +109,12 @@ export class IosVoiceControlBridge { stop(): void { ++this.generation this.capturing = this.finalizing = false + this.source = null this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() this.finishCompletion() } handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void { - if (!this.capturing && !this.finalizing) return + if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return this.stop(); this.setStatus(message); this.emitEnd() } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } @@ -112,10 +134,11 @@ export class IosVoiceControlBridge { } else if (event.kind === 'status') this.setStatus(event.message ?? '') else if (event.kind === 'audio') { this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`) - if (this.capturing) this.setStatus(`Listening on glasses… ${event.seconds?.toFixed(0)}s`) + if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`) } else if (event.kind === 'finishing') { this.stopPushToTalk(); this.emitEnd() } else if (event.kind === 'ended') { + this.source = null this.capturing = this.finalizing = false this.clearAudioTimer(); this.releaseMicrophone() this.finishCompletion() diff --git a/app/phone-ui/README.md b/app/phone-ui/README.md new file mode 100644 index 00000000..c8d13a6c --- /dev/null +++ b/app/phone-ui/README.md @@ -0,0 +1,33 @@ +# Main-screen UI + +Android and iOS share `remote-controls.xml`, `remote-controls.css`, and +`RemoteControlsViewModel`: tab order and selection, screen-size choices, +brightness/Auto settings, watch labels, ring markings, and microphone/keyboard +buttons. Import the stylesheet at the top of each platform's application CSS +(before any rules), so the NativeScript CSS pipeline includes it. + +Android's `MainViewModel` extends that shared model and connects it to +`dashboardController`. iOS loads the same XML with `Builder.load` and connects +it to `IosPreviewController`. Keep platform-specific transport, page lifecycle, +and navigation in those adapters. Android uses NativeScript gesture events; +iOS feeds raw touches to `PhoneGestureRecognizer`, so the discrete pad gesture +bindings in the XML are Android-only to avoid delivering each gesture twice. + +The iOS shell uses the same portrait preview-above-controls layout and 345-DIP +landscape controls column. It respects safe areas, uses the same overflow menu +labels/order, and retains device diagnostics under the connection status. +Conversations and Record screen are disabled on iOS because their storage and +recording services are currently Android-only. Take screenshot opens the iOS +share sheet. Text settings still use native dialogs. Keyboard input uses the +shared `keyboard-input-panel.xml` multiline editor and shell keyboard session, +with separate Agent and App destinations when available. iOS keeps the editor +above the software keyboard in both orientations. + +Voice input uses the glasses microphone when connected and the phone's default +audio input in preview mode. The phone path requires both Speech Recognition +and Microphone permissions and stops when Faceclaw leaves the foreground; +connected-glasses capture retains its existing background behavior. + +The iOS microphone and keyboard image assets are 3x rasterizations of the +Android vector drawables with the same names; regenerate them from those +vectors if the icon designs change. diff --git a/app/phone-ui/keyboard-input-panel.xml b/app/phone-ui/keyboard-input-panel.xml new file mode 100644 index 00000000..bd36734d --- /dev/null +++ b/app/phone-ui/keyboard-input-panel.xml @@ -0,0 +1,18 @@ + +