From c9cf29e5c073b6cb1a542e0106a1db768ffdaf25 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sat, 24 Oct 2026 00:18:04 -0700 Subject: [PATCH 01/34] Add an iOS Spotify integration --- App_Resources/iOS/build.xcconfig | 2 +- App_Resources/iOS/src/FaceclawAppleMusic.m | 26 +- App_Resources/iOS/src/FaceclawCrypto.h | 4 + App_Resources/iOS/src/FaceclawCrypto.m | 21 + App_Resources/iOS/src/FaceclawGraphics.h | 7 + App_Resources/iOS/src/FaceclawGraphics.m | 38 ++ .../iOS/src/FaceclawLoopbackRedirect.h | 16 + .../iOS/src/FaceclawLoopbackRedirect.m | 128 ++++ App_Resources/iOS/src/FaceclawWebSignIn.h | 21 + App_Resources/iOS/src/FaceclawWebSignIn.m | 110 ++++ App_Resources/iOS/src/module.modulemap | 2 + CHANGELOG | 1 + PRIVACY | 14 + app/apps/music/music-app.ts | 63 +- app/apps/music/spotify-menu.ts | 107 ++++ app/assistant/system-tools.ts | 48 +- app/native/media-browser.ios.ts | 45 +- app/native/media-controller.ios.ts | 200 +++++- app/native/media-types.ts | 24 +- app/native/spotify-auth.ts | 273 +++++++++ app/native/spotify-library.ts | 157 +++++ app/native/spotify-player.ts | 570 ++++++++++++++++++ app/native/spotify-service.ts | 159 +++++ app/native/spotify-web-api.ts | 474 +++++++++++++++ app/native/spotify.ios.ts | 63 ++ app/native/spotify.ts | 9 + app/ui/dashboard-settings.ts | 16 + tests/ios-apple-music.test.cjs | 10 +- tests/ios-calendar.test.cjs | 1 + tests/ios-loopback-redirect-native.cjs | 18 + tests/ios-spotify.test.cjs | 530 ++++++++++++++++ tests/native/ios-loopback-redirect-test.m | 69 +++ 32 files changed, 3176 insertions(+), 50 deletions(-) create mode 100644 App_Resources/iOS/src/FaceclawLoopbackRedirect.h create mode 100644 App_Resources/iOS/src/FaceclawLoopbackRedirect.m create mode 100644 App_Resources/iOS/src/FaceclawWebSignIn.h create mode 100644 App_Resources/iOS/src/FaceclawWebSignIn.m create mode 100644 app/apps/music/spotify-menu.ts create mode 100644 app/native/spotify-auth.ts create mode 100644 app/native/spotify-library.ts create mode 100644 app/native/spotify-player.ts create mode 100644 app/native/spotify-service.ts create mode 100644 app/native/spotify-web-api.ts create mode 100644 app/native/spotify.ios.ts create mode 100644 app/native/spotify.ts create mode 100644 tests/ios-loopback-redirect-native.cjs create mode 100644 tests/ios-spotify.test.cjs create mode 100644 tests/native/ios-loopback-redirect-test.m diff --git a/App_Resources/iOS/build.xcconfig b/App_Resources/iOS/build.xcconfig index 03805280..13dda0ff 100644 --- a/App_Resources/iOS/build.xcconfig +++ b/App_Resources/iOS/build.xcconfig @@ -3,7 +3,7 @@ TARGETED_DEVICE_FAMILY = 1; CODE_SIGN_STYLE = Automatic; // Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team. -OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer +OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; diff --git a/App_Resources/iOS/src/FaceclawAppleMusic.m b/App_Resources/iOS/src/FaceclawAppleMusic.m index d6b8cef0..9d8e63ce 100644 --- a/App_Resources/iOS/src/FaceclawAppleMusic.m +++ b/App_Resources/iOS/src/FaceclawAppleMusic.m @@ -1,4 +1,5 @@ #import "FaceclawAppleMusic.h" +#import "FaceclawGraphics.h" #import #import #import @@ -219,30 +220,7 @@ static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *sub - (NSData *)albumArt:(NSInteger)maxSize { if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil; MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork; - UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)]; - CGImageRef source = image.CGImage; - if (!source) return nil; - double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source); - if (sourceWidth <= 0 || sourceHeight <= 0) return nil; - double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight)); - NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale)); - NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4]; - uint8_t *bytes = packet.mutableBytes; - bytes[0] = width & 255; bytes[1] = (width >> 8) & 255; - bytes[2] = height & 255; bytes[3] = (height >> 8) & 255; - uint8_t *pixels = bytes + 4; - CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); - // Memory order B, G, R, A: one little-endian ARGB word per pixel. - CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space, - kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst); - CGColorSpaceRelease(space); - if (!context) return nil; - CGContextSetInterpolationQuality(context, kCGInterpolationHigh); - CGContextDrawImage(context, CGRectMake(0, 0, width, height), source); - CGContextRelease(context); - // Composited over black (the display's see-through background); now opaque. - for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255; - return packet; + return [FaceclawGraphics bgraPacket:[artwork imageWithSize:CGSizeMake(maxSize, maxSize)].CGImage maxSize:maxSize]; } #pragma mark - Queue diff --git a/App_Resources/iOS/src/FaceclawCrypto.h b/App_Resources/iOS/src/FaceclawCrypto.h index c0eb02ba..cdd6ffc4 100644 --- a/App_Resources/iOS/src/FaceclawCrypto.h +++ b/App_Resources/iOS/src/FaceclawCrypto.h @@ -2,4 +2,8 @@ @interface FaceclawCrypto : NSObject + (NSString *)hmacSha256:(NSString *)secret message:(NSString *)message; + (NSString *)sha256:(NSData *)data; +/** byteCount bytes from the system CSPRNG, base64url without padding (an OAuth state or PKCE verifier). */ ++ (NSString *)randomBase64Url:(NSInteger)byteCount; +/** SHA-256 of the text's UTF-8, base64url without padding (a PKCE S256 code challenge). */ ++ (NSString *)sha256Base64Url:(NSString *)text; @end diff --git a/App_Resources/iOS/src/FaceclawCrypto.m b/App_Resources/iOS/src/FaceclawCrypto.m index 6814fb3c..2e968a65 100644 --- a/App_Resources/iOS/src/FaceclawCrypto.m +++ b/App_Resources/iOS/src/FaceclawCrypto.m @@ -1,6 +1,15 @@ #import "FaceclawCrypto.h" #import #import +#import + +static NSString *Base64Url(NSData *data) { + NSString *base64 = [data base64EncodedStringWithOptions:0]; + base64 = [base64 stringByReplacingOccurrencesOfString:@"+" withString:@"-"]; + base64 = [base64 stringByReplacingOccurrencesOfString:@"/" withString:@"_"]; + return [base64 stringByReplacingOccurrencesOfString:@"=" withString:@""]; +} + @implementation FaceclawCrypto + (NSString *)hmacSha256:(NSString *)secret message:(NSString *)message { NSData *key = [secret dataUsingEncoding:NSUTF8StringEncoding]; @@ -16,4 +25,16 @@ for (NSUInteger i = 0; i < sizeof(digest); i++) [hex appendFormat:@"%02x", digest[i]]; return hex; } ++ (NSString *)randomBase64Url:(NSInteger)byteCount { + if (byteCount <= 0 || byteCount > 1024) return nil; + NSMutableData *bytes = [NSMutableData dataWithLength:(NSUInteger)byteCount]; + if (SecRandomCopyBytes(kSecRandomDefault, bytes.length, bytes.mutableBytes) != errSecSuccess) return nil; + return Base64Url(bytes); +} ++ (NSString *)sha256Base64Url:(NSString *)text { + NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding]; + unsigned char digest[CC_SHA256_DIGEST_LENGTH]; + CC_SHA256(data.bytes, (CC_LONG)data.length, digest); + return Base64Url([NSData dataWithBytes:digest length:sizeof(digest)]); +} @end diff --git a/App_Resources/iOS/src/FaceclawGraphics.h b/App_Resources/iOS/src/FaceclawGraphics.h index 64614bdf..2f71ddcd 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.h +++ b/App_Resources/iOS/src/FaceclawGraphics.h @@ -7,4 +7,11 @@ + (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size; + (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height; + (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight; +/** + * [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), + * composited over black; the input of the shared Kotlin gray tone curve and dither. + */ ++ (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize; +/** bgraPacket of encoded image bytes (JPEG, PNG, ...); nil when undecodable. */ ++ (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize; @end diff --git a/App_Resources/iOS/src/FaceclawGraphics.m b/App_Resources/iOS/src/FaceclawGraphics.m index 84e7a453..cd8f3f16 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.m +++ b/App_Resources/iOS/src/FaceclawGraphics.m @@ -120,4 +120,42 @@ CGColorSpaceRelease(space); return gray; } ++ (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize { + if (!image || maxSize <= 0 || maxSize > 1024) return nil; + double sourceWidth = CGImageGetWidth(image), sourceHeight = CGImageGetHeight(image); + if (sourceWidth <= 0 || sourceHeight <= 0) return nil; + double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight)); + NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale)); + NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4]; + uint8_t *bytes = packet.mutableBytes; + bytes[0] = width & 255; bytes[1] = (width >> 8) & 255; + bytes[2] = height & 255; bytes[3] = (height >> 8) & 255; + uint8_t *pixels = bytes + 4; + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + // Memory order B, G, R, A: one little-endian ARGB word per pixel. + CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space, + kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst); + CGColorSpaceRelease(space); + if (!context) return nil; + CGContextSetInterpolationQuality(context, kCGInterpolationHigh); + CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); + CGContextRelease(context); + // Composited over black (the display's see-through background); now opaque. + for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255; + return packet; +} ++ (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize { + if (!data.length || maxSize <= 0 || maxSize > 1024) return nil; + CGImageSourceRef source = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL); + if (!source) return nil; + NSDictionary *options = @{(id)kCGImageSourceCreateThumbnailFromImageAlways: @YES, + (id)kCGImageSourceCreateThumbnailWithTransform: @YES, + (id)kCGImageSourceThumbnailMaxPixelSize: @(maxSize)}; + CGImageRef image = CGImageSourceCreateThumbnailAtIndex(source, 0, (__bridge CFDictionaryRef)options); + CFRelease(source); + if (!image) return nil; + NSData *packet = [self bgraPacket:image maxSize:maxSize]; + CGImageRelease(image); + return packet; +} @end diff --git a/App_Resources/iOS/src/FaceclawLoopbackRedirect.h b/App_Resources/iOS/src/FaceclawLoopbackRedirect.h new file mode 100644 index 00000000..f5c81d5f --- /dev/null +++ b/App_Resources/iOS/src/FaceclawLoopbackRedirect.h @@ -0,0 +1,16 @@ +#import + +/** + * Catches an OAuth redirect to http://127.0.0.1: (the loopback redirect that + * providers such as Spotify accept from native apps): listens on the loopback interface + * only, answers the browser's request for the path with a short "return to Faceclaw" + * page and hands back the full redirect URL, query included. Any other path (a favicon) + * gets a 404. Foundation-only, so the macOS test builds it as is. + */ +@interface FaceclawLoopbackRedirect : NSObject +/** Called on the main queue with each redirect to the path. */ +@property(nonatomic, copy) void (^redirectHandler)(NSString *url); +/** Start listening; nil on success, otherwise why not (e.g. another app holds the port). */ +- (NSString *)startOnPort:(NSInteger)port path:(NSString *)path; +- (void)stop; +@end diff --git a/App_Resources/iOS/src/FaceclawLoopbackRedirect.m b/App_Resources/iOS/src/FaceclawLoopbackRedirect.m new file mode 100644 index 00000000..3f9fca8a --- /dev/null +++ b/App_Resources/iOS/src/FaceclawLoopbackRedirect.m @@ -0,0 +1,128 @@ +#import "FaceclawLoopbackRedirect.h" +#import +#import +#import +#import +#import +#import + +/** Request heads beyond this are not a browser's redirect; stop reading. */ +static const NSUInteger kMaxRequestBytes = 16384; + +static NSString *const kDonePage = + @"" + @"Faceclaw" + @"" + @"

Signed in

You can return to Faceclaw.

"; + +@interface FaceclawLoopbackRedirect () +@property(nonatomic, strong) dispatch_queue_t queue; +@property(nonatomic, strong) dispatch_source_t acceptSource; +@property(nonatomic) NSInteger port; +@property(nonatomic, copy) NSString *path; +@end + +@implementation FaceclawLoopbackRedirect +- (instancetype)init { + if ((self = [super init])) { + _queue = dispatch_queue_create("com.faceclaw.loopback-redirect", DISPATCH_QUEUE_SERIAL); + } + return self; +} + +- (void)dealloc { + [self stop]; +} + +- (NSString *)startOnPort:(NSInteger)port path:(NSString *)path { + [self stop]; + if (port <= 0 || port > 65535 || ![path hasPrefix:@"/"]) return @"Invalid redirect address"; + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return [NSString stringWithFormat:@"Could not open a socket (errno %d)", errno]; + int yes = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct sockaddr_in address = {0}; + address.sin_len = sizeof(address); + address.sin_family = AF_INET; + address.sin_port = htons((uint16_t)port); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (bind(fd, (struct sockaddr *)&address, sizeof(address)) != 0 || listen(fd, 8) != 0) { + int error = errno; + close(fd); + return error == EADDRINUSE + ? [NSString stringWithFormat:@"Another app is using port %ld; close it and try again", (long)port] + : [NSString stringWithFormat:@"Could not listen on port %ld (errno %d)", (long)port, error]; + } + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + self.port = port; + self.path = path; + dispatch_source_t source = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, (uintptr_t)fd, 0, self.queue); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(source, ^{ + int client = accept(fd, NULL, NULL); + if (client < 0) return; + FaceclawLoopbackRedirect *strongSelf = weakSelf; + if (strongSelf) [strongSelf serve:client]; + else close(client); + }); + dispatch_source_set_cancel_handler(source, ^{ close(fd); }); + self.acceptSource = source; + dispatch_resume(source); + return nil; +} + +- (void)stop { + if (self.acceptSource) dispatch_source_cancel(self.acceptSource); + self.acceptSource = nil; +} + +/** Read one request head and answer it; runs on the receiver's queue (blocking, with timeouts). */ +- (void)serve:(int)client { + // BSD sockets inherit O_NONBLOCK from the listener. + fcntl(client, F_SETFL, fcntl(client, F_GETFL) & ~O_NONBLOCK); + int yes = 1; + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes)); + struct timeval timeout = { .tv_sec = 3, .tv_usec = 0 }; + setsockopt(client, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + + NSMutableData *head = [NSMutableData data]; + NSData *terminator = [@"\r\n\r\n" dataUsingEncoding:NSASCIIStringEncoding]; + char buffer[2048]; + while (head.length < kMaxRequestBytes) { + ssize_t count = recv(client, buffer, sizeof(buffer), 0); + if (count <= 0) break; + [head appendBytes:buffer length:(NSUInteger)count]; + if ([head rangeOfData:terminator options:0 range:NSMakeRange(0, head.length)].location != NSNotFound) break; + } + NSString *text = [[NSString alloc] initWithData:head encoding:NSUTF8StringEncoding] ?: @""; + NSArray *requestLine = [[text componentsSeparatedByString:@"\r\n"].firstObject componentsSeparatedByString:@" "]; + NSString *target = requestLine.count >= 3 && [requestLine[0] isEqualToString:@"GET"] ? requestLine[1] : nil; + BOOL matched = target && [[target componentsSeparatedByString:@"?"].firstObject isEqualToString:self.path]; + [self respond:client status:matched ? @"200 OK" : @"404 Not Found" body:matched ? kDonePage : @"Not found"]; + close(client); + if (!matched) return; + + NSString *url = [NSString stringWithFormat:@"http://127.0.0.1:%ld%@", (long)self.port, target]; + void (^handler)(NSString *) = self.redirectHandler; + dispatch_async(dispatch_get_main_queue(), ^{ + if (handler) handler(url); + }); +} + +- (void)respond:(int)client status:(NSString *)status body:(NSString *)body { + NSData *bodyData = [body dataUsingEncoding:NSUTF8StringEncoding]; + NSString *headers = [NSString stringWithFormat: + @"HTTP/1.1 %@\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: %lu\r\n" + @"Cache-Control: no-store\r\nConnection: close\r\n\r\n", status, (unsigned long)bodyData.length]; + NSMutableData *response = [[headers dataUsingEncoding:NSASCIIStringEncoding] mutableCopy]; + [response appendData:bodyData]; + const uint8_t *bytes = response.bytes; + NSUInteger sent = 0; + while (sent < response.length) { + ssize_t count = send(client, bytes + sent, response.length - sent, 0); + if (count <= 0) break; + sent += (NSUInteger)count; + } +} +@end diff --git a/App_Resources/iOS/src/FaceclawWebSignIn.h b/App_Resources/iOS/src/FaceclawWebSignIn.h new file mode 100644 index 00000000..fd0274a4 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawWebSignIn.h @@ -0,0 +1,21 @@ +#import + +/** + * OAuth sign-in in the system browser sheet (ASWebAuthenticationSession, which shares + * Safari's cookies, so a provider the user is already signed in to needs only a consent + * tap) for providers that redirect to a loopback address: a FaceclawLoopbackRedirect + * catches the redirect, then the sheet is closed. Presenting needs Faceclaw in the + * foreground, so a sign-in started from the glasses waits until the user opens the app. + * One sign-in at a time; starting another cancels the first. Call from the main thread. + */ +@interface FaceclawWebSignIn : NSObject ++ (instancetype)shared; +/** + * completion(redirectUrl, nil) once the provider redirects to http://127.0.0.1:; + * (nil, "cancelled") when the user closes the sheet or cancel is called; (nil, message) + * when the sheet or the receiver cannot start. Called once, on the main queue. + */ +- (void)signIn:(NSString *)authorizeUrl port:(NSInteger)port path:(NSString *)path + completion:(void (^)(NSString *redirectUrl, NSString *error))completion; +- (void)cancel; +@end diff --git a/App_Resources/iOS/src/FaceclawWebSignIn.m b/App_Resources/iOS/src/FaceclawWebSignIn.m new file mode 100644 index 00000000..3c1d76e2 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawWebSignIn.m @@ -0,0 +1,110 @@ +#import "FaceclawWebSignIn.h" +#import "FaceclawLoopbackRedirect.h" +#import +#import + +/** Never reached: the loopback receiver ends the session before any custom-scheme redirect. */ +static NSString *const kUnusedCallbackScheme = @"faceclaw-signin"; + +@interface FaceclawWebSignIn () +@property(nonatomic, strong) FaceclawLoopbackRedirect *receiver; +@property(nonatomic, strong) ASWebAuthenticationSession *session; +@property(nonatomic, copy) void (^completion)(NSString *redirectUrl, NSString *error); +/** The sign-in page waiting for Faceclaw to come to the foreground. */ +@property(nonatomic, copy) NSURL *pendingUrl; +@property(nonatomic, strong) id activeObserver; +@end + +@implementation FaceclawWebSignIn ++ (instancetype)shared { + static FaceclawWebSignIn *instance; + static dispatch_once_t once; + dispatch_once(&once, ^{ instance = [FaceclawWebSignIn new]; }); + return instance; +} + +- (void)signIn:(NSString *)authorizeUrl port:(NSInteger)port path:(NSString *)path + completion:(void (^)(NSString *, NSString *))completion { + [self finishWithUrl:nil error:@"cancelled"]; + NSURL *url = [NSURL URLWithString:authorizeUrl]; + if (!url || ![url.scheme isEqualToString:@"https"]) { + if (completion) completion(nil, @"Invalid sign-in address"); + return; + } + FaceclawLoopbackRedirect *receiver = [FaceclawLoopbackRedirect new]; + NSString *error = [receiver startOnPort:port path:path]; + if (error) { + if (completion) completion(nil, error); + return; + } + __weak typeof(self) weakSelf = self; + receiver.redirectHandler = ^(NSString *redirectUrl) { + [weakSelf finishWithUrl:redirectUrl error:nil]; + }; + self.receiver = receiver; + self.completion = completion; + if (UIApplication.sharedApplication.applicationState == UIApplicationStateActive) { + [self present:url]; + return; + } + self.pendingUrl = url; + self.activeObserver = [NSNotificationCenter.defaultCenter addObserverForName:UIApplicationDidBecomeActiveNotification + object:nil queue:NSOperationQueue.mainQueue usingBlock:^(NSNotification *note) { + FaceclawWebSignIn *strongSelf = weakSelf; + NSURL *pending = strongSelf.pendingUrl; + if (!pending) return; + [strongSelf removeActiveObserver]; + [strongSelf present:pending]; + }]; +} + +- (void)cancel { + [self finishWithUrl:nil error:@"cancelled"]; +} + +- (void)present:(NSURL *)url { + __weak typeof(self) weakSelf = self; + ASWebAuthenticationSession *session = [[ASWebAuthenticationSession alloc] initWithURL:url + callbackURLScheme:kUnusedCallbackScheme completionHandler:^(NSURL *callbackURL, NSError *error) { + // The sheet closed: the user dismissed it, or finishing already cancelled it. + BOOL dismissed = !error || ([error.domain isEqualToString:ASWebAuthenticationSessionErrorDomain] + && error.code == ASWebAuthenticationSessionErrorCodeCanceledLogin); + [weakSelf finishWithUrl:callbackURL.absoluteString + error:callbackURL ? nil : dismissed ? @"cancelled" : error.localizedDescription]; + }]; + session.presentationContextProvider = self; + session.prefersEphemeralWebBrowserSession = NO; + self.session = session; + if (![session start]) [self finishWithUrl:nil error:@"Could not open the sign-in page"]; +} + +/** Report the outcome once; tear down the receiver and close the sheet. */ +- (void)finishWithUrl:(NSString *)url error:(NSString *)error { + void (^completion)(NSString *, NSString *) = self.completion; + ASWebAuthenticationSession *session = self.session; + self.completion = nil; + self.session = nil; + self.pendingUrl = nil; + [self removeActiveObserver]; + [self.receiver stop]; + self.receiver = nil; + // Re-enters through the session's completion handler, which then finds nothing to report. + [session cancel]; + if (completion) completion(url, url ? nil : error); +} + +- (void)removeActiveObserver { + if (self.activeObserver) [NSNotificationCenter.defaultCenter removeObserver:self.activeObserver]; + self.activeObserver = nil; +} + +- (ASPresentationAnchor)presentationAnchorForWebAuthenticationSession:(ASWebAuthenticationSession *)session { + for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { + if (![scene isKindOfClass:UIWindowScene.class]) continue; + NSArray *windows = ((UIWindowScene *)scene).windows; + for (UIWindow *window in windows) if (window.isKeyWindow) return window; + if (windows.firstObject) return windows.firstObject; + } + return [UIWindow new]; +} +@end diff --git a/App_Resources/iOS/src/module.modulemap b/App_Resources/iOS/src/module.modulemap index 02935284..597281a8 100644 --- a/App_Resources/iOS/src/module.modulemap +++ b/App_Resources/iOS/src/module.modulemap @@ -11,5 +11,7 @@ module FaceclawGraphics { header "FaceclawEvenHubWebView.h" header "FaceclawCrypto.h" header "FaceclawQrScanner.h" + header "FaceclawLoopbackRedirect.h" + header "FaceclawWebSignIn.h" export * } diff --git a/CHANGELOG b/CHANGELOG index 8b412589..d2ec8fb6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,7 @@ * Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover * Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view * Add an Apple Music integration on iOS +* Add a Spotify integration on iOS by voice (needs your own Spotify developer client ID and Spotify Premium) ### Bug fixes diff --git a/PRIVACY b/PRIVACY index fd667185..4d937543 100644 --- a/PRIVACY +++ b/PRIVACY @@ -153,6 +153,20 @@ Apple's on iOS) for the name of the place. Open-Meteo: https://open-meteo.com/en/terms +## Spotify + +On iOS, the Music app, Glanceboard and the voice assistant can show and +control Spotify through Spotify's Web API, once the user has created their own +Spotify developer app and signed in. Signing in happens on Spotify's website in +the system's sign-in sheet; Faceclaw never sees the password, and stores the +resulting sign-in tokens on the phone. While Spotify is shown or used, +Faceclaw requests the user's playback state, queue, library and search results +from Spotify and sends it playback commands, directly from the phone. This is +subject to Spotify's privacy policy. + + Spotify: https://www.spotify.com/legal/privacy-policy/ + + ## EvenHub The EvenHub app store is subject to Even Realities' privacy policy: diff --git a/app/apps/music/music-app.ts b/app/apps/music/music-app.ts index f35a1130..f5bb8964 100644 --- a/app/apps/music/music-app.ts +++ b/app/apps/music/music-app.ts @@ -8,8 +8,10 @@ import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics"; import { mediaControllerBridge, type MediaControllerState, type MediaQueueItem } from "../../native/media-controller"; import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser"; +import { spotifyService } from "../../native/spotify"; import { MediaBrowseLayer } from "./media-browse"; import { MusicSettingsLayer } from "./music-settings"; +import { spotifyMenuItem } from "./spotify-menu"; import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers"; import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; @@ -33,6 +35,8 @@ const QUEUE_X = 204; /** Horizontal inset of row text from its selection box. */ const LIST_TEXT_INSET = 6; const PLAYLIST_ACTION_INDEX = 1; +/** Minimum room for the dim artist after a queue row's title; less and it is left off. */ +const QUEUE_SUBTITLE_MIN_WIDTH = 40; type MusicAction = | { kind: "play-pause"; label: string; enabled: boolean } @@ -40,14 +44,17 @@ type MusicAction = | { kind: "next"; label: string; enabled: boolean } | { kind: "volume"; label: string; enabled: boolean } | { kind: "playlist"; label: string; enabled: boolean } - | { kind: "browse"; label: string; enabled: boolean }; + | { kind: "browse"; label: string; enabled: boolean } + /** A player-specific action from the state's customActions (Spotify's shuffle / repeat / like). */ + | { kind: "custom"; label: string; enabled: boolean; action: string }; type FocusColumn = "actions" | "playlist"; /** * Music controller app: metadata + album art + transport controls for the * active Android media session (any player that publishes one) or, on iOS, - * the Apple Music system player, plus the player's queue when it exposes one + * the Apple Music system player or Spotify (signed in from the window menu), + * plus the player's queue when it exposes one * (scroll to a track, click to jump). With no session, the front page lists * the apps whose library can be browsed (click one to open its * MediaBrowseLayer). @@ -110,7 +117,7 @@ class MusicAppLayer implements Layer { const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh); this.libraryAppsFresh = true; this.libraryMenu.setItems(apps); - image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180); + image.drawText(font, 24, 16, global.isIOS ? media.status || "Nothing is playing." : "No active media session.", 180); const prompt = global.isIOS ? "Browse your library, or start playback on the phone." : apps.length @@ -121,6 +128,11 @@ class MusicAppLayer implements Layer { image.drawText(font, 24, y, line, 150); y += lineStep(font); } + // Why the last command failed (e.g. playing from the Spotify library with no device open). + for (const line of media.notice ? wrapText(font, media.notice, width - 48) : []) { + image.drawText(font, 24, y, line, 200); + y += lineStep(font); + } if (apps.length) { const listTop = y + 6; const listHeight = height - listTop - LIST_BOTTOM_MARGIN; @@ -156,12 +168,13 @@ class MusicAppLayer implements Layer { if (media.album) { image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150); } + // The player line, or for a few seconds why a command failed. image.drawText( font, META_X, metaTop + 4 + (titleSlots + 2) * metaStep, - truncateText(font, media.appName || media.packageName, metaWidth), - 110, + truncateText(font, media.notice || media.appName || media.packageName, metaWidth), + media.notice ? 200 : 110, ); this.drawProgress(image, media, metaWidth); @@ -212,7 +225,16 @@ class MusicAppLayer implements Layer { private drawQueueItem({ image, item, x, y, width, height, selected }: MenuDrawArgs): void { const font = getDefaultSmallFont(); const label = `${item.active ? "> " : " "}${item.title || "(untitled)"}`; - image.drawText(font, x + LIST_TEXT_INSET, centeredTextY(font, y, height), truncateText(font, label, width - 2 * LIST_TEXT_INSET), selected ? 255 : 200); + const textWidth = width - 2 * LIST_TEXT_INSET; + const textY = centeredTextY(font, y, height); + const title = truncateText(font, label, textWidth); + image.drawText(font, x + LIST_TEXT_INSET, textY, title, selected ? 255 : 200); + // The artist, dimmer, in whatever room the title leaves. + const subtitleX = font.measureText(`${label} `); + if (item.subtitle && title === label && textWidth - subtitleX >= QUEUE_SUBTITLE_MIN_WIDTH) { + image.drawText(font, x + LIST_TEXT_INSET + subtitleX, textY, + truncateText(font, item.subtitle, textWidth - subtitleX), selected ? 150 : 110); + } } async handleInput(event: InputEvent, ctx: LayerContext): Promise { @@ -278,6 +300,7 @@ class MusicAppLayer implements Layer { else if (action.kind === "previous") await mediaControllerBridge.skipPrevious(); else if (action.kind === "next") await mediaControllerBridge.skipNext(); else if (action.kind === "browse") this.openBrowse(ctx); + else if (action.kind === "custom") await mediaControllerBridge.sendCustomAction?.(action.action); else if (action.kind === "volume") { ctx.stack.push(new VolumeModalLayer(mediaControllerBridge.getMediaVolumePercent())); } else if (action.kind === "playlist") { @@ -342,6 +365,12 @@ class MusicAppLayer implements Layer { { kind: "volume", label: volume >= 0 ? `Volume (${volume})` : "Volume", enabled: volume >= 0 }, { kind: "next", label: "Next track", enabled: media.canSkipNext }, { kind: "previous", label: "Previous track", enabled: media.canSkipPrevious }, + ...(mediaControllerBridge.sendCustomAction ? media.customActions ?? [] : []).map((custom): MusicAction => ({ + kind: "custom", + label: custom.name, + enabled: true, + action: custom.action, + })), ]; } @@ -496,14 +525,20 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin iconLetter: "M", icon: "music", closeable: true, - // Music settings choose among Android media apps; iOS has only Apple Music. - menuItems: () => global.isIOS ? [] : [{ - label: "Music settings", - onSelect: (ctx) => { - ctx.stack.pop(); - ctx.stack.push(new MusicSettingsLayer()); - }, - }], + // Music settings choose among Android media apps; iOS has Apple Music, plus Spotify's sign-in here. + menuItems: () => { + if (!global.isIOS) { + return [{ + label: "Music settings", + onSelect: (ctx) => { + ctx.stack.pop(); + ctx.stack.push(new MusicSettingsLayer()); + }, + }]; + } + const spotify = spotifyService(); + return spotify ? [spotifyMenuItem(spotify)] : []; + }, actions: options.actions, baseLayer: new MusicRootLayer(musicLayer), submitFrame: options.submitFrame, diff --git a/app/apps/music/spotify-menu.ts b/app/apps/music/spotify-menu.ts new file mode 100644 index 00000000..9b3805b1 --- /dev/null +++ b/app/apps/music/spotify-menu.ts @@ -0,0 +1,107 @@ +import type { GrayImage } from "../../graphics/image"; +import { isValidSpotifyClientId, SPOTIFY_DASHBOARD_URL, SPOTIFY_REDIRECT_URI } from "../../native/spotify-auth"; +import type { SpotifyService } from "../../native/spotify-service"; +import { spotifyClientIdSetting, textSettingMenuItem } from "../../ui/dashboard-settings"; +import { type InputEvent } from "../../ui/gestures"; +import { type LayerContext, type PaintBelow } from "../../ui/layers"; +import { submenuItem, TextPageLayer, type MenuItem } from "../../ui/menu"; +import { WindowMenuLayer } from "../../ui/window-menu"; +import { TextViewerLayer } from "../files/text-viewer"; + +const SETUP_HELP = [ + "Faceclaw controls Spotify through Spotify's Web API, which needs a Spotify developer app of your own. Creating one is free; Spotify requires its owner to have Premium, and controlling playback needs Premium too.", + "", + `1. On a computer, open ${SPOTIFY_DASHBOARD_URL.replace("https://", "")} and sign in with your Spotify account.`, + "2. Create an app. Any name and description will do.", + `3. Add this Redirect URI exactly: ${SPOTIFY_REDIRECT_URI}`, + "4. Under \"Which API/SDKs are you planning to use?\", tick Web API, then save.", + "5. Open the app's settings, copy its Client ID, and enter it as Client ID here.", + "6. Choose Sign in, then finish on your phone: Faceclaw opens Spotify's sign-in page there.", + "", + "Spotify plays on whichever of your devices is open (this phone, a computer, a speaker). To start music when nothing is playing, Spotify must be open on one of them.", + "", + "Others can sign in with your client ID once you add their Spotify email under User Management in the dashboard (up to five people).", +].join("\n"); + +/** The Music app menu's Spotify row (iOS): sign in, sign out, client ID, setup help. */ +export function spotifyMenuItem(service: SpotifyService): MenuItem { + return submenuItem("Spotify", (ctx) => ctx.stack.push(new SpotifyMenuLayer(service)), { + description: "Sign in to Spotify to show and control it from the glasses.", + }); +} + +/** + * Rows rebuilt every paint and input from the account and sign-in progress, + * which change asynchronously. A WindowMenuLayer, so the shell's system menu + * closes it like the window menu it opens from. + */ +class SpotifyMenuLayer extends WindowMenuLayer { + private unsubscribe: (() => void) | null = null; + + constructor(private readonly service: SpotifyService) { + super("Spotify", []); + } + + paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { + this.unsubscribe ??= this.service.onChange(() => ctx.actions.requestRender()); + this.setItems(this.rows()); + return super.paint(ctx, paintBelow); + } + + async handleInput(event: InputEvent, ctx: LayerContext): Promise { + this.setItems(this.rows()); + await super.handleInput(event, ctx); + } + + onRemoved(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + } + + private rows(): MenuItem[] { + const service = this.service; + const account = service.account(); + const status = service.signInStatus(); + const help: MenuItem = { + label: "Setup help", + onSelect: (ctx) => ctx.stack.push(new TextViewerLayer(SETUP_HELP, "Spotify setup")), + }; + if (account) { + return [ + info(`Signed in as ${account.displayName || account.userId}`), + ...(account.product && account.product !== "premium" ? [info("Playback control needs Premium")] : []), + { label: "Sign out", onSelect: () => service.signOut() }, + textSettingMenuItem(spotifyClientIdSetting), + help, + ]; + } + if (status.phase === "waiting") { + return [ + info("Finish signing in on the phone"), + { label: "Cancel sign-in", onSelect: () => service.cancelSignIn() }, + help, + ]; + } + const failure: MenuItem[] = status.error + ? [{ + label: "Sign-in failed (details)", + onSelect: (ctx) => ctx.stack.push(new TextPageLayer("Spotify sign-in failed", status.error)), + }] + : []; + return [ + { + label: "Sign in", + description: "Opens Spotify's sign-in page on the phone.", + disabled: () => !isValidSpotifyClientId(spotifyClientIdSetting.get()), + onSelect: () => void service.signIn(), + }, + textSettingMenuItem(spotifyClientIdSetting), + help, + ...failure, + ]; + } +} + +function info(label: string): MenuItem { + return { label, disabled: true, onSelect: () => {} }; +} diff --git a/app/assistant/system-tools.ts b/app/assistant/system-tools.ts index 9baacf56..b6745d83 100644 --- a/app/assistant/system-tools.ts +++ b/app/assistant/system-tools.ts @@ -1,6 +1,7 @@ import { readUpcomingEventsAsync, type CalendarEvent } from "../native/calendar"; import { hasCalendarPermission } from "../native/calendar-permissions"; import { mediaControllerBridge } from "../native/media-controller"; +import { spotifyService } from "../native/spotify"; import { dismissNotification, readActiveNotifications } from "../native/notification-icons"; import { shell } from "../ui/shell/shell"; import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry"; @@ -93,7 +94,11 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void inputSchema: { type: "object", properties: {}, additionalProperties: false }, proactive: true, }, - () => ok(describeNowPlaying()), + async () => { + // Spotify's state is polled only while something shows it. + await mediaControllerBridge.refresh?.(); + return ok(describeNowPlaying()); + }, ); registry.registerSystemTool( @@ -103,6 +108,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void inputSchema: { type: "object", properties: {}, additionalProperties: false }, }, async () => { + await mediaControllerBridge.refresh?.(); const state = mediaControllerBridge.snapshot(); if (!state.canPlayPause) return err("No media session can be controlled right now."); await mediaControllerBridge.playPause(); @@ -117,6 +123,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void inputSchema: { type: "object", properties: {}, additionalProperties: false }, }, async () => { + await mediaControllerBridge.refresh?.(); const state = mediaControllerBridge.snapshot(); if (!state.canSkipNext) return err("The current media session can't skip to the next track."); await mediaControllerBridge.skipNext(); @@ -124,6 +131,45 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void }, ); + const spotify = spotifyService(); + if (spotify) { + registry.register({ + spec: { + name: "media.spotify_play", + description: + "Search Spotify and start playing the best match on the user's Spotify (this phone or another of their Spotify devices). Use for requests like \"play Daft Punk\" (artist), \"play the album Discovery\" (album) or \"play Harder Better Faster Stronger\" (track).", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "What to search for, e.g. an artist, album, song or playlist name." }, + kind: { + type: "string", + enum: ["track", "album", "artist", "playlist"], + description: "What kind of thing to play (default track).", + }, + }, + required: ["query"], + additionalProperties: false, + }, + availability: "always", + // Search, then play (with a device lookup when none is active). + timeoutMs: 20_000, + }, + handler: async (args) => { + const query = String(args?.query ?? "").trim(); + if (!query) return err("media.spotify_play requires a query"); + const kind = ["track", "album", "artist", "playlist"].includes(args?.kind) ? args.kind : "track"; + try { + return ok(`Playing ${await spotify.playSearch(query, kind)} on Spotify.`); + } catch (error) { + return err(String((error as Error)?.message ?? error)); + } + }, + // Offered once signed in from the Music app menu. + isAvailable: () => spotify.account() !== null, + }); + } + // Android notification tools are still platform-specific. if (global.isIOS) return; diff --git a/app/native/media-browser.ios.ts b/app/native/media-browser.ios.ts index ac1ded94..5b5497ae 100644 --- a/app/native/media-browser.ios.ts +++ b/app/native/media-browser.ios.ts @@ -6,6 +6,9 @@ import { type MediaBrowserApp, type MediaBrowserBridge, } from "./media-types"; +import { spotifyService } from "./spotify.ios"; +import type { SpotifyLibrary } from "./spotify-library"; +import { SPOTIFY_PACKAGE } from "./spotify-player"; export type { MediaBrowseItem, MediaBrowserApp } from "./media-types"; @@ -30,7 +33,7 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge { connect(app: MediaBrowserApp): Promise { this.disconnect(); - if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS")); + if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Not an Apple Music library")); if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed")); this.connected = true; return Promise.resolve("root"); @@ -67,4 +70,42 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge { } } -export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge(); +/** Apple Music's library, plus the Spotify library once signed in. */ +export class IosMediaBrowserBridge implements MediaBrowserBridge { + private current: MediaBrowserBridge | null = null; + + constructor( + private readonly appleMusic: AppleMusicBrowserBridge, + private readonly spotify: () => SpotifyLibrary | null, + ) {} + + listBrowsableApps(): MediaBrowserApp[] { + return [...this.appleMusic.listBrowsableApps(), ...(this.spotify()?.listBrowsableApps() ?? [])]; + } + + connect(app: MediaBrowserApp): Promise { + this.disconnect(); + const library = app.packageName === SPOTIFY_PACKAGE ? this.spotify() : this.appleMusic; + if (!library) return Promise.reject(new Error(`${app.appName} can't be browsed`)); + this.current = library; + return library.connect(app); + } + + browse(parentId: string): Promise { + return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected")); + } + + playFromMediaId(mediaId: string): void { + this.current?.playFromMediaId(mediaId); + } + + disconnect(): void { + this.current?.disconnect(); + this.current = null; + } +} + +export const mediaBrowserBridge: MediaBrowserBridge = new IosMediaBrowserBridge( + new AppleMusicBrowserBridge(), + () => spotifyService()?.library ?? null, +); diff --git a/app/native/media-controller.ios.ts b/app/native/media-controller.ios.ts index 1200c7ce..e13a4252 100644 --- a/app/native/media-controller.ios.ts +++ b/app/native/media-controller.ios.ts @@ -11,6 +11,8 @@ import { type MediaPlaybackState, type MediaQueueItem, } from "./media-types"; +import { spotifyService } from "./spotify.ios"; +import type { SpotifyPlayer } from "./spotify-player"; export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types"; @@ -88,7 +90,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge { onStateChange(listener: (state: MediaControllerState) => void): () => void { void this.start(); this.stateListeners.add(listener); - this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS); + this.pollTimer ??= setInterval(() => this.publish(), POLL_MS); listener(this.snapshot()); return () => { this.stateListeners.delete(listener); @@ -142,7 +144,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge { if (this.hasAccess()) return Promise.resolve(true); this.pendingAccess ??= this.askForAccess().finally(() => { this.pendingAccess = null; - this.refresh(); + this.publish(); }); return this.pendingAccess; } @@ -187,12 +189,12 @@ export class AppleMusicControllerBridge implements MediaControllerBridge { const music = FaceclawAppleMusic.shared(); if (!this.bound) { this.bound = true; - music.changeHandler = () => this.refresh(); + music.changeHandler = () => this.publish(); } return music; } - private refresh(): void { + private publish(): void { if (!this.stateListeners.size) return; const state = this.snapshot(); const signature = stateSignature(state); @@ -228,4 +230,192 @@ export class AppleMusicControllerBridge implements MediaControllerBridge { } } -export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge(); +export type IosMediaSource = "apple-music" | "spotify"; + +/** + * Which player to follow: the one playing; with both playing, the current + * choice (no flapping); with neither, the one that played last, and before + * either has played this run, Spotify when it reports a session (its player + * reports only a recently active one, while the system Music player keeps + * its last item for days). + */ +export function chooseMediaSource( + current: IosMediaSource, + appleMusic: MediaControllerState, + spotify: MediaControllerState | null, + appleMusicLastPlayingAtMs: number, + spotifyLastPlayingAtMs: number, +): IosMediaSource { + if (!spotify?.accessEnabled) return "apple-music"; + if (!appleMusic.accessEnabled) return "spotify"; + const applePlaying = appleMusic.playbackState === "playing"; + const spotifyPlaying = spotify.playbackState === "playing"; + if (applePlaying && spotifyPlaying) return current; + if (applePlaying) return "apple-music"; + if (spotifyPlaying) return "spotify"; + if (appleMusicLastPlayingAtMs !== spotifyLastPlayingAtMs) { + return spotifyLastPlayingAtMs > appleMusicLastPlayingAtMs ? "spotify" : "apple-music"; + } + return spotify.available ? "spotify" : "apple-music"; +} + +/** + * The iOS Music bridge: Apple Music (the system player), plus Spotify once + * signed in (its Web API: playback on this phone or any Spotify Connect + * device). The state and every command follow one of them, by + * chooseMediaSource. + */ +export class IosMediaControllerBridge implements MediaControllerBridge { + readonly accessPrompt = + "Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone. For Spotify, sign in from the app menu instead."; + readonly accessLabel: string; + readonly accessAction: string; + private source: IosMediaSource = "apple-music"; + private appleMusicLastPlayingAtMs = 0; + private readonly listeners = new Set<(state: MediaControllerState) => void>(); + private unsubscribeSources: Array<() => void> = []; + /** The source and its artVersion the composite artVersion was bumped for. */ + private artKey = ""; + private artVersion = 0; + + constructor( + private readonly appleMusic: AppleMusicControllerBridge, + private readonly spotify: () => SpotifyPlayer | null, + private readonly now: () => number = Date.now, + ) { + this.accessLabel = appleMusic.accessLabel; + this.accessAction = appleMusic.accessAction; + } + + get skipRebuildsQueue(): boolean { + return this.active().skipRebuildsQueue; + } + + onStateChange(listener: (state: MediaControllerState) => void): () => void { + if (!this.listeners.size) { + const update = () => this.emit(); + const spotify = this.spotify(); + this.unsubscribeSources = [this.appleMusic.onStateChange(update), ...(spotify ? [spotify.onStateChange(update)] : [])]; + } + this.listeners.add(listener); + listener(this.snapshot()); + return () => { + this.listeners.delete(listener); + if (this.listeners.size) return; + for (const unsubscribe of this.unsubscribeSources) unsubscribe(); + this.unsubscribeSources = []; + }; + } + + snapshot(): MediaControllerState { + const appleMusic = this.appleMusic.snapshot(); + const spotify = this.spotify()?.snapshot() ?? null; + this.select(appleMusic, spotify); + const state = this.source === "spotify" && spotify ? spotify : appleMusic; + // Changes with the source as well as with that source's art. + const artKey = `${this.source}:${state.artVersion}`; + if (artKey !== this.artKey) { + this.artKey = artKey; + this.artVersion++; + } + return { + ...state, + status: state.available ? state.status : idleStatus(appleMusic, spotify, state), + artVersion: this.artVersion, + // A Spotify command can fail while Apple Music is the one shown (playing from the Spotify library). + notice: spotify?.notice || state.notice || "", + }; + } + + async start(): Promise { + await this.appleMusic.start(); + } + + async stop(): Promise { + await this.appleMusic.stop(); + } + + playPause(): Promise { + return this.active().playPause(); + } + + skipNext(): Promise { + return this.active().skipNext(); + } + + skipPrevious(): Promise { + return this.active().skipPrevious(); + } + + /** Apple Music's media-library permission (what requestAccess asks for); Spotify signs in from the Music menu. */ + hasAccess(): boolean { + return this.appleMusic.hasAccess(); + } + + requestAccess(): Promise { + return this.appleMusic.requestAccess(); + } + + skipToQueueItem(id: number): Promise { + return this.active().skipToQueueItem(id); + } + + getMediaVolumePercent(): number { + return this.active().getMediaVolumePercent(); + } + + setMediaVolumePercent(volumePercent: number): void { + this.active().setMediaVolumePercent(volumePercent); + } + + getAlbumArt(maxSize: number): GrayImage | null { + return this.active().getAlbumArt(maxSize); + } + + getQueue(): MediaQueueItem[] { + return this.active().getQueue(); + } + + async sendCustomAction(action: string): Promise { + await this.active().sendCustomAction?.(action); + } + + async refresh(): Promise { + await this.spotify()?.refresh(); + } + + private active(): MediaControllerBridge { + const spotify = this.spotify(); + this.select(this.appleMusic.snapshot(), spotify?.snapshot() ?? null); + return this.source === "spotify" && spotify ? spotify : this.appleMusic; + } + + private select(appleMusic: MediaControllerState, spotify: MediaControllerState | null): void { + if (appleMusic.playbackState === "playing") this.appleMusicLastPlayingAtMs = this.now(); + this.source = chooseMediaSource(this.source, appleMusic, spotify, this.appleMusicLastPlayingAtMs, + this.spotify()?.lastPlayingAtMs ?? 0); + } + + /** + * Forward every source update: each source already drops repeats, and + * Spotify also signals changes that are not in the state (its queue). + */ + private emit(): void { + if (!this.listeners.size) return; + const state = this.snapshot(); + for (const listener of this.listeners) listener(state); + } +} + +/** The no-session line: which players were checked. */ +function idleStatus(appleMusic: MediaControllerState, spotify: MediaControllerState | null, shown: MediaControllerState): string { + if (appleMusic.accessEnabled && spotify?.accessEnabled) return "Nothing is playing."; + return shown.status; +} + +export const appleMusicControllerBridge = new AppleMusicControllerBridge(); + +export const mediaControllerBridge: MediaControllerBridge = new IosMediaControllerBridge( + appleMusicControllerBridge, + () => spotifyService()?.player ?? null, +); diff --git a/app/native/media-types.ts b/app/native/media-types.ts index 84449b30..086f11e6 100644 --- a/app/native/media-types.ts +++ b/app/native/media-types.ts @@ -3,7 +3,8 @@ import type { GrayImage } from "../graphics/image"; /** * Shapes shared by the media bridges: media-controller.ts / media-browser.ts * drive Android media sessions and MediaBrowserServices; their .ios.ts - * counterparts drive Apple Music through the MediaPlayer framework. + * counterparts drive Apple Music through the MediaPlayer framework and + * Spotify through its Web API. */ /** The player behind the iOS bridges: the system Music app. */ @@ -13,9 +14,19 @@ export const APPLE_MUSIC_PACKAGE = "com.apple.Music"; export type MediaQueueItem = { id: number; title: string; + /** Usually the artist; absent when the player gives none. */ + subtitle?: string; active: boolean; }; +/** A player-specific action beyond the transport controls (Spotify's shuffle, repeat, like). */ +export type MediaCustomAction = { + /** Stable id to send back with sendCustomAction. */ + action: string; + /** The row label, naming the current state where there is one ("Shuffle: on"). */ + name: string; +}; + export type MediaPlaybackState = | "notification-access-required" | "idle" @@ -47,6 +58,10 @@ export type MediaControllerState = { * players re-publish the same strings with a different bitmap. */ artVersion: number; + /** Player-specific actions (see sendCustomAction); absent when the player offers none. */ + customActions?: MediaCustomAction[]; + /** A short-lived message about a command that failed (Spotify needs Premium, ...); absent or "" otherwise. */ + notice?: string; }; export const DEFAULT_MEDIA_STATE: MediaControllerState = { @@ -113,6 +128,13 @@ export interface MediaControllerBridge { getAlbumArt(maxSize: number): GrayImage | null; /** The player's queue (playlist), empty when the player exposes none. */ getQueue(): MediaQueueItem[]; + /** Run one of the state's customActions. */ + sendCustomAction?(action: string): Promise; + /** + * Re-read the player now, for one-off readers (the assistant) when no + * listener keeps the state current; absent when snapshots are always live. + */ + refresh?(): Promise; } export type MediaBrowserApp = { diff --git a/app/native/spotify-auth.ts b/app/native/spotify-auth.ts new file mode 100644 index 00000000..956a5141 --- /dev/null +++ b/app/native/spotify-auth.ts @@ -0,0 +1,273 @@ +import { getStringSetting, setStringSetting } from "./settings-store"; +import { buildQuery, SpotifyApiError, SpotifyClient, type SpotifyFetch, type SpotifyTokenSource } from "./spotify-web-api"; + +/** + * Spotify sign-in: OAuth authorization code with PKCE against the user's own + * client ID (no client secret), redirected to a loopback address that the + * phone catches (FaceclawWebSignIn on iOS). Spotify accepts http only for + * loopback IP literals, and custom-scheme redirects fail with "insecure + * redirect URI" for apps created since 2025, so the redirect is + * http://127.0.0.1 on a fixed port the user registers in the dashboard. + * + * The tokens are stored with the client ID they were issued to; changing the + * client ID setting signs out. + */ + +export const SPOTIFY_ACCOUNTS_BASE = "https://accounts.spotify.com"; +export const SPOTIFY_REDIRECT_PORT = 43823; +export const SPOTIFY_REDIRECT_PATH = "/spotify-callback"; +export const SPOTIFY_REDIRECT_URI = `http://127.0.0.1:${SPOTIFY_REDIRECT_PORT}${SPOTIFY_REDIRECT_PATH}`; +export const SPOTIFY_DASHBOARD_URL = "https://developer.spotify.com/dashboard"; + +export const SPOTIFY_CLIENT_ID_KEY = "music.spotify.clientId"; +export const SPOTIFY_ACCOUNT_KEY = "music.spotify.account"; + +export const SPOTIFY_SCOPES = [ + "user-read-playback-state", + "user-modify-playback-state", + "user-read-currently-playing", + "user-read-recently-played", + "playlist-read-private", + "playlist-read-collaborative", + "user-library-read", + "user-library-modify", + "user-follow-read", +]; + +/** Refresh this long before Spotify's stated expiry (an hour), so a token never lapses mid-request. */ +const EXPIRY_MARGIN_MS = 60_000; + +export type SpotifyAccount = { + clientId: string; + accessToken: string; + refreshToken: string; + /** Epoch ms. */ + expiresAtMs: number; + userId: string; + displayName: string; + /** "premium", "free", ...; playback control needs premium. */ + product: string; +}; + +export type SpotifyTokens = { accessToken: string; refreshToken: string; expiresAtMs: number }; + +/** The phone side of a sign-in: PKCE material and the browser sheet that catches the loopback redirect. */ +export type WebSignIn = { + /** A fresh PKCE verifier and its S256 challenge. */ + createPkce(): { verifier: string; challenge: string }; + randomState(): string; + /** Show the provider's page; resolves to the redirect URL, rejects with "cancelled" or why it failed. */ + open(authorizeUrl: string, port: number, path: string): Promise; + cancel(): void; +}; + +export function spotifyClientId(): string { + return getStringSetting(SPOTIFY_CLIENT_ID_KEY, "").trim(); +} + +export function isValidSpotifyClientId(value: string): boolean { + return /^[0-9a-f]{32}$/i.test(value.trim()); +} + +/** The stored sign-in, or null when signed out or signed in with another client ID. */ +export function readSpotifyAccount(): SpotifyAccount | null { + let raw: any; + try { + raw = JSON.parse(getStringSetting(SPOTIFY_ACCOUNT_KEY, "") || "null"); + } catch { + return null; + } + if (!raw || typeof raw.refreshToken !== "string" || !raw.refreshToken) return null; + const clientId = spotifyClientId(); + if (!clientId || raw.clientId !== clientId) return null; + return { + clientId, + accessToken: String(raw.accessToken ?? ""), + refreshToken: raw.refreshToken, + expiresAtMs: Number(raw.expiresAtMs) || 0, + userId: String(raw.userId ?? ""), + displayName: String(raw.displayName ?? ""), + product: String(raw.product ?? ""), + }; +} + +export function saveSpotifyAccount(account: SpotifyAccount | null): void { + setStringSetting(SPOTIFY_ACCOUNT_KEY, account ? JSON.stringify(account) : ""); +} + +export function spotifyAuthorizeUrl(clientId: string, codeChallenge: string, state: string): string { + return `${SPOTIFY_ACCOUNTS_BASE}/authorize${buildQuery({ + response_type: "code", + client_id: clientId, + scope: SPOTIFY_SCOPES.join(" "), + redirect_uri: SPOTIFY_REDIRECT_URI, + state, + code_challenge_method: "S256", + code_challenge: codeChallenge, + })}`; +} + +/** The authorization code from Spotify's redirect; throws with a message for the glasses otherwise. */ +export function parseSpotifyRedirect(url: string, expectedState: string): string { + const params = parseQueryString(url); + if (params.error) { + throw new Error(params.error === "access_denied" ? "Spotify sign-in was declined." : `Spotify sign-in failed: ${params.error}`); + } + if (params.state !== expectedState) throw new Error("Spotify sign-in failed: the reply didn't match this sign-in. Try again."); + if (!params.code) throw new Error("Spotify sign-in failed: no authorization code in the reply."); + return params.code; +} + +/** Query parameters of a URL (not URLSearchParams, which the NativeScript runtime mishandles). */ +export function parseQueryString(url: string): Record { + const query = url.split("#")[0]!.split("?").slice(1).join("?"); + const params: Record = {}; + for (const part of query.split("&")) { + if (!part) continue; + const separator = part.indexOf("="); + const key = separator < 0 ? part : part.slice(0, separator); + const value = separator < 0 ? "" : part.slice(separator + 1); + try { + params[decodeURIComponent(key.replace(/\+/g, " "))] = decodeURIComponent(value.replace(/\+/g, " ")); + } catch { + // Malformed escapes: skip the parameter. + } + } + return params; +} + +/** POST to Spotify's token endpoint (authorization_code or refresh_token grant). */ +export async function requestSpotifyTokens( + fetchImpl: SpotifyFetch, + params: Record, + nowMs: number, +): Promise { + let status = 0; + let body = ""; + try { + const response = await fetchImpl(`${SPOTIFY_ACCOUNTS_BASE}/api/token`, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json" }, + body: buildQuery(params).slice(1), + }); + status = response.status; + body = await response.text(); + } catch (error) { + throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0); + } + let parsed: any = null; + try { + parsed = JSON.parse(body); + } catch { + // Reported below. + } + if (status >= 200 && status < 300 && typeof parsed?.access_token === "string") { + const expiresIn = Number(parsed.expires_in); + return { + accessToken: parsed.access_token, + // Spotify may rotate the refresh token; "" keeps the current one. + refreshToken: typeof parsed.refresh_token === "string" ? parsed.refresh_token : "", + expiresAtMs: nowMs + (Number.isFinite(expiresIn) && expiresIn > 0 ? expiresIn : 3600) * 1000, + }; + } + const error = typeof parsed?.error === "string" ? parsed.error : ""; + const description = typeof parsed?.error_description === "string" ? parsed.error_description : ""; + if (error === "invalid_grant" && params.grant_type === "refresh_token") { + throw new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, error); + } + if (error === "invalid_client") { + throw new SpotifyApiError("Spotify doesn't recognize the client ID. Check it under Spotify in the Music app menu.", status, error); + } + throw new SpotifyApiError(`Spotify sign-in failed: ${description || error || `HTTP ${status}`}`, status, error); +} + +/** + * Access tokens for the stored account: the current one until it nears + * expiry, then one refresh shared by every caller waiting on it. A refresh + * Spotify rejects (revoked access) signs out. + */ +export class SpotifySession implements SpotifyTokenSource { + private refreshing: Promise | null = null; + + constructor( + private readonly fetchImpl: SpotifyFetch, + private readonly now: () => number = Date.now, + ) {} + + accessToken(forceRefresh: boolean): Promise { + const account = readSpotifyAccount(); + if (!account) return Promise.reject(new SpotifyApiError("Not signed in to Spotify.", 401, "signed_out")); + if (!forceRefresh && account.accessToken && account.expiresAtMs - EXPIRY_MARGIN_MS > this.now()) { + return Promise.resolve(account.accessToken); + } + this.refreshing ??= this.refresh(account).finally(() => { + this.refreshing = null; + }); + return this.refreshing; + } + + private async refresh(account: SpotifyAccount): Promise { + let tokens: SpotifyTokens; + try { + tokens = await requestSpotifyTokens(this.fetchImpl, { + grant_type: "refresh_token", + refresh_token: account.refreshToken, + client_id: account.clientId, + }, this.now()); + } catch (error) { + if (error instanceof SpotifyApiError && error.reason === "invalid_grant" && readSpotifyAccount()?.refreshToken === account.refreshToken) { + saveSpotifyAccount(null); + } + throw error; + } + // A sign-out or another sign-in while this was in flight wins. + if (readSpotifyAccount()?.refreshToken !== account.refreshToken) { + throw new SpotifyApiError("Spotify sign-in changed; try again.", 401, "signed_out"); + } + saveSpotifyAccount({ + ...account, + accessToken: tokens.accessToken, + refreshToken: tokens.refreshToken || account.refreshToken, + expiresAtMs: tokens.expiresAtMs, + }); + return tokens.accessToken; + } +} + +/** + * The whole sign-in: the browser sheet, the code exchange and the profile + * lookup; stores and returns the account. Rejects with "cancelled" when the + * user closes the sheet. + */ +export async function signInToSpotify( + fetchImpl: SpotifyFetch, + webSignIn: WebSignIn, + now: () => number = Date.now, +): Promise { + const clientId = spotifyClientId(); + if (!isValidSpotifyClientId(clientId)) throw new Error("Enter your Spotify client ID first."); + const { verifier, challenge } = webSignIn.createPkce(); + const state = webSignIn.randomState(); + const redirect = await webSignIn.open(spotifyAuthorizeUrl(clientId, challenge, state), SPOTIFY_REDIRECT_PORT, SPOTIFY_REDIRECT_PATH); + const code = parseSpotifyRedirect(redirect, state); + const tokens = await requestSpotifyTokens(fetchImpl, { + grant_type: "authorization_code", + code, + redirect_uri: SPOTIFY_REDIRECT_URI, + client_id: clientId, + code_verifier: verifier, + }, now()); + const user = await new SpotifyClient(fetchImpl, { accessToken: async () => tokens.accessToken }).me(); + const account: SpotifyAccount = { + clientId, + accessToken: tokens.accessToken, + refreshToken: tokens.refreshToken, + expiresAtMs: tokens.expiresAtMs, + userId: user.id, + displayName: user.displayName, + product: user.product, + }; + if (!account.refreshToken) throw new Error("Spotify sign-in failed: no refresh token in the reply."); + saveSpotifyAccount(account); + return account; +} diff --git a/app/native/spotify-library.ts b/app/native/spotify-library.ts new file mode 100644 index 00000000..a45989c8 --- /dev/null +++ b/app/native/spotify-library.ts @@ -0,0 +1,157 @@ +import type { MediaBrowseItem, MediaBrowserApp, MediaBrowserBridge } from "./media-types"; +import { SPOTIFY_APP_NAME, SPOTIFY_PACKAGE, type SpotifyPlayer } from "./spotify-player"; +import type { SpotifyClient, SpotifyItem, SpotifyPlayRequest } from "./spotify-web-api"; + +export const SPOTIFY_BROWSER_APP: MediaBrowserApp = { packageName: SPOTIFY_PACKAGE, serviceClass: "", appName: SPOTIFY_APP_NAME }; + +const ROOT_ROWS: MediaBrowseItem[] = [ + { mediaId: "recent", title: "Recently played", subtitle: "", browsable: true, playable: false }, + { mediaId: "playlists", title: "Playlists", subtitle: "", browsable: true, playable: false }, + { mediaId: "liked", title: "Liked Songs", subtitle: "", browsable: true, playable: true }, + { mediaId: "albums", title: "Albums", subtitle: "", browsable: true, playable: false }, + { mediaId: "artists", title: "Artists", subtitle: "", browsable: true, playable: false }, +]; + +/** Separates a track's URI from the context it plays in ("" for none, "liked" for Liked Songs). */ +const TRACK_CONTEXT_SEPARATOR = "|"; + +export type SpotifyLibraryDeps = { + client: SpotifyClient; + player: SpotifyPlayer; + isSignedIn(): boolean; + /** The signed-in user's id (Liked Songs' context URI names it). */ + userId(): string; +}; + +/** + * The signed-in user's Spotify library as a browse tree (Web API): + * + * root -> recent | playlists | liked | albums | artists + * playlist:, album: -> tracks (browsable and playable) + * artist: -> albums + * track:| -> plays the track within its context + * + * Only playlists the user owns or collaborates on can list their tracks + * (development-mode client IDs, since February 2026); others are play-only. + */ +export class SpotifyLibrary implements MediaBrowserBridge { + private connected = false; + /** Bumped on every connect/disconnect; answers to older requests are dropped. */ + private generation = 0; + + constructor(private readonly deps: SpotifyLibraryDeps) {} + + listBrowsableApps(): MediaBrowserApp[] { + return this.deps.isSignedIn() ? [SPOTIFY_BROWSER_APP] : []; + } + + connect(app: MediaBrowserApp): Promise { + this.disconnect(); + if (app.packageName !== SPOTIFY_PACKAGE) return Promise.reject(new Error("Not a Spotify library")); + if (!this.deps.isSignedIn()) return Promise.reject(new Error("Sign in under Spotify in the Music app menu first.")); + this.connected = true; + return Promise.resolve("root"); + } + + async browse(parentId: string): Promise { + if (!this.connected) throw new Error("Not connected"); + const generation = this.generation; + const rows = await this.list(parentId); + if (generation !== this.generation) throw new Error("Disconnected"); + return rows; + } + + playFromMediaId(mediaId: string): void { + if (!this.connected) return; + const request = this.playRequest(mediaId); + if (request) void this.deps.player.startPlayback(request); + } + + disconnect(): void { + this.connected = false; + this.generation++; + } + + /** What playing a browse row starts; null for rows that don't play. */ + playRequest(mediaId: string): SpotifyPlayRequest | null { + if (mediaId === "liked") return { contextUri: this.likedContextUri() }; + const [kind, ...rest] = mediaId.split(":"); + const id = rest.join(":"); + switch (kind) { + case "playlist": + case "album": + case "artist": + return id ? { contextUri: `spotify:${kind}:${id}` } : null; + case "track": { + const separator = id.lastIndexOf(TRACK_CONTEXT_SEPARATOR); + const uri = separator < 0 ? id : id.slice(0, separator); + const context = separator < 0 ? "" : id.slice(separator + 1); + if (!uri) return null; + if (context === "liked") return { contextUri: this.likedContextUri(), offsetUri: uri }; + return context ? { contextUri: context, offsetUri: uri } : { uris: [uri] }; + } + default: + return null; + } + } + + private async list(parentId: string): Promise { + const client = this.deps.client; + if (parentId === "root") return ROOT_ROWS; + if (parentId === "recent") { + // The history repeats tracks; keep each one's latest play. + const seen = new Set(); + return (await client.recentlyPlayed()) + .filter((played) => !seen.has(played.item.uri) && seen.add(played.item.uri)) + .map((played) => trackRow(played.item, played.contextUri)); + } + if (parentId === "playlists") { + const userId = this.deps.userId(); + return (await client.playlists()).map((playlist) => { + const listable = playlist.collaborative || playlist.ownerId === userId; + return { + mediaId: `playlist:${playlist.id}`, + title: playlist.name, + subtitle: listable ? "" : playlist.ownerName, + browsable: listable, + playable: true, + }; + }); + } + if (parentId === "liked") return (await client.savedTracks()).map((item) => trackRow(item, "liked")); + if (parentId === "albums") return (await client.savedAlbums()).map(albumRow); + if (parentId === "artists") { + return (await client.followedArtists()).map((artist) => ({ + mediaId: `artist:${artist.id}`, + title: artist.name, + subtitle: "", + browsable: true, + playable: true, + })); + } + const [kind, ...rest] = parentId.split(":"); + const id = rest.join(":"); + if (kind === "playlist") return (await client.playlistItems(id)).map((item) => trackRow(item, `spotify:playlist:${id}`)); + if (kind === "album") return (await client.albumTracks(id)).map((item) => trackRow(item, `spotify:album:${id}`)); + if (kind === "artist") return (await client.artistAlbums(id)).map(albumRow); + throw new Error(`Unknown Spotify library item: ${parentId}`); + } + + private likedContextUri(): string { + return `spotify:user:${this.deps.userId()}:collection`; + } +} + +function trackRow(item: SpotifyItem, contextUri: string): MediaBrowseItem { + return { + mediaId: `track:${item.uri}${contextUri ? `${TRACK_CONTEXT_SEPARATOR}${contextUri}` : ""}`, + title: item.name, + subtitle: item.artists, + browsable: false, + playable: true, + }; +} + +function albumRow(album: { id: string; name: string; artists: string }): MediaBrowseItem { + return { mediaId: `album:${album.id}`, title: album.name, subtitle: album.artists, browsable: true, playable: true }; +} diff --git a/app/native/spotify-player.ts b/app/native/spotify-player.ts new file mode 100644 index 00000000..becfe13f --- /dev/null +++ b/app/native/spotify-player.ts @@ -0,0 +1,570 @@ +import type { GrayImage } from "../graphics/image"; +import { + DEFAULT_MEDIA_STATE, + interpolateMediaState, + type MediaControllerBridge, + type MediaControllerState, + type MediaCustomAction, + type MediaQueueItem, +} from "./media-types"; +import { + SpotifyApiError, + type SpotifyClient, + type SpotifyDevice, + type SpotifyItem, + type SpotifyPlayback, + type SpotifyPlayRequest, + type SpotifyRepeat, +} from "./spotify-web-api"; + +export const SPOTIFY_APP_NAME = "Spotify"; +/** The iOS app's bundle id; Spotify's Android package is com.spotify.music. */ +export const SPOTIFY_PACKAGE = "com.spotify.client"; + +/** + * Poll intervals while something listens. The Web API has no push channel, so + * a track change shows up within one interval (sooner at a track's end, which + * is polled for directly); requests count against the client ID's quota. + */ +const POLL_PLAYING_MS = 5_000; +const POLL_PAUSED_MS = 10_000; +const POLL_IDLE_MS = 20_000; +const POLL_ERROR_MS = 30_000; +/** Re-read soon after a command: Spotify takes a moment to report its effect. */ +const SETTLE_POLL_MS = 600; +const SETTLE_POLLS = 2; +const NOTICE_MS = 8_000; +/** Re-read the queue this often while the Music app shows it, besides on every track change. */ +const QUEUE_MAX_AGE_MS = 30_000; +/** At most this often, even on track changes (and after a failed read). */ +const QUEUE_MIN_INTERVAL_MS = 5_000; +/** Album art: the smallest of Spotify's sizes (640/300/64) at least this wide. */ +const ART_MIN_WIDTH = 128; + +const REPEAT_NEXT: Record = { off: "context", context: "track", track: "off" }; +const REPEAT_LABEL: Record = { off: "Repeat: off", context: "Repeat: all", track: "Repeat: one" }; + +export type SpotifyPlayerDeps = { + client: SpotifyClient; + isSignedIn(): boolean; + /** Fetch an album-art image's encoded bytes; null on failure. */ + download(url: string): Promise; + /** Encoded image bytes to the shared photo tone, fitted within maxSize; null when undecodable. */ + decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null; + /** The phone's output volume, for playback on this phone (Spotify won't set a phone's volume). */ + systemVolume: { get(): number; set(percent: number): void }; + now?: () => number; +}; + +/** + * Spotify Connect playback (on this phone or any of the user's devices) + * through the Web API: state from polling GET /me/player while something + * listens, commands as API calls whose failures (no device, no Premium) + * become a short notice in the state. + */ +export class SpotifyPlayer implements MediaControllerBridge { + readonly accessPrompt = "Sign in to Spotify from the Music app menu to show and control Spotify."; + readonly accessLabel = "Spotify sign-in needed"; + readonly accessAction = "open the menu"; + /** Jumping skips forward through the queue, so the chosen track ends up first. */ + readonly skipRebuildsQueue = true; + /** When Spotify was last seen playing (epoch ms); 0 = not this run. */ + lastPlayingAtMs = 0; + + private readonly now: () => number; + private readonly listeners = new Set<(state: MediaControllerState) => void>(); + private playback: SpotifyPlayback | null = null; + private fetchedAtMs = 0; + private pollTimer: ReturnType | null = null; + private polling: Promise | null = null; + private pollAgain = false; + private settlePolls = 0; + private lastSignature = ""; + private notice = ""; + private noticeUntilMs = 0; + private noticeTimer: ReturnType | null = null; + private upcoming: SpotifyItem[] = []; + /** The item the queue was read with; "" = never read. */ + private queueCurrentUri = ""; + /** When the queue was last read, or a read failed. */ + private queueFetchedAtMs = 0; + private queueLoading = false; + private art: { url: string; bytes: Uint8Array | null; decoded: Map } | null = null; + private artVersion = 0; + private liked: { uri: string; value: boolean | null } | null = null; + private volumeTarget: number | null = null; + private volumeSending = false; + + constructor(private readonly deps: SpotifyPlayerDeps) { + this.now = deps.now ?? Date.now; + } + + onStateChange(listener: (state: MediaControllerState) => void): () => void { + this.listeners.add(listener); + listener(this.snapshot()); + if (this.listeners.size === 1) this.schedulePoll(0); + return () => { + this.listeners.delete(listener); + if (!this.listeners.size && !this.settlePolls) this.clearPollTimer(); + }; + } + + snapshot(): MediaControllerState { + const notice = this.currentNotice(); + if (!this.deps.isSignedIn()) { + return { ...DEFAULT_MEDIA_STATE, status: "Not signed in to Spotify.", notice }; + } + const playback = this.playback; + if (!playback) { + return { + ...DEFAULT_MEDIA_STATE, + accessEnabled: true, + packageName: SPOTIFY_PACKAGE, + appName: SPOTIFY_APP_NAME, + status: "Nothing is playing on Spotify.", + notice, + }; + } + const item = playback.item; + const restricted = playback.device?.restricted === true; + const allowed = (action: string) => !restricted && !playback.disallows.includes(action); + const state: MediaControllerState = { + available: true, + accessEnabled: true, + packageName: SPOTIFY_PACKAGE, + appName: spotifyAppName(playback.device), + playbackState: playback.playing ? "playing" : "paused", + title: item?.name ?? (playback.playingType === "ad" ? "Advertisement" : ""), + artist: item?.artists ?? "", + album: item?.album ?? "", + positionMs: item ? playback.progressMs : -1, + durationMs: item?.durationMs ?? -1, + playbackSpeed: playback.playing ? 1 : 0, + status: "", + canPlayPause: allowed(playback.playing ? "pausing" : "resuming"), + canSkipNext: allowed("skipping_next"), + canSkipPrevious: allowed("skipping_prev"), + artVersion: this.artVersion, + customActions: restricted ? [] : this.customActions(playback), + notice, + }; + return interpolateMediaState(state, this.fetchedAtMs, this.now()); + } + + async start(): Promise {} + + async stop(): Promise {} + + async playPause(): Promise { + const playback = this.playback; + if (!playback?.playing) { + await this.startPlayback({}); + return; + } + await this.command(() => this.deps.client.pause(), () => { + playback.progressMs = this.snapshot().positionMs; + playback.playing = false; + this.fetchedAtMs = this.now(); + }); + } + + async skipNext(): Promise { + await this.command(() => this.deps.client.next()); + } + + async skipPrevious(): Promise { + await this.command(() => this.deps.client.previous()); + } + + /** Signed in; whether anything is playing is in the state. */ + hasAccess(): boolean { + return this.deps.isSignedIn(); + } + + /** Sign-in happens from the Music app menu, which shows its progress. */ + async requestAccess(): Promise { + return this.hasAccess(); + } + + /** + * Queue ids are positions in getQueue (0 = the current item). The Web API + * has no jump, so this skips forward that many times. + */ + async skipToQueueItem(id: number): Promise { + const steps = Math.round(id); + if (steps <= 0 || steps > this.upcoming.length) return; + await this.command(async () => { + for (let step = 0; step < steps; step++) await this.deps.client.next(); + }, () => { + const target = this.upcoming[steps - 1]; + if (this.playback && target) { + this.playback.item = target; + this.playback.progressMs = 0; + this.fetchedAtMs = this.now(); + this.upcoming = this.upcoming.slice(steps); + this.queueCurrentUri = target.uri; + this.itemChanged(target); + } + }); + } + + getMediaVolumePercent(): number { + const device = this.playback?.device ?? null; + if (!usesDeviceVolume(device)) return this.deps.systemVolume.get(); + return this.volumeTarget ?? device!.volumePercent; + } + + setMediaVolumePercent(volumePercent: number): void { + const device = this.playback?.device ?? null; + const volume = Math.max(0, Math.min(100, Math.round(volumePercent))); + if (!usesDeviceVolume(device)) { + this.deps.systemVolume.set(volume); + return; + } + // Scrolling sends a value per step: keep one request in flight and send the latest after it. + device!.volumePercent = volume; + this.volumeTarget = volume; + if (!this.volumeSending) void this.sendVolume(device!.id); + } + + getAlbumArt(maxSize: number): GrayImage | null { + const art = this.art; + if (!art?.bytes) return null; + const size = Math.round(maxSize); + if (!art.decoded.has(size)) art.decoded.set(size, this.deps.decodeArt(art.bytes, size)); + return art.decoded.get(size) ?? null; + } + + /** + * The current item, then what Spotify will play next. Read on demand (the + * Music app asks while it paints), so nothing else pays for the request; a + * stale list stays up, minus the items already played, until the new one + * arrives. + */ + getQueue(): MediaQueueItem[] { + const current = this.playback?.item; + if (!current) return []; + const age = this.now() - this.queueFetchedAtMs; + if (!this.queueLoading && (age > QUEUE_MAX_AGE_MS || (this.queueCurrentUri !== current.uri && age > QUEUE_MIN_INTERVAL_MS))) { + void this.loadQueue(); + } + if (!this.queueCurrentUri) return []; + if (this.queueCurrentUri !== current.uri) { + const index = this.upcoming.findIndex((item) => item.uri === current.uri); + if (index >= 0) { + this.upcoming = this.upcoming.slice(index + 1); + this.queueCurrentUri = current.uri; + } + } + return [current, ...this.upcoming].map((item, index) => ({ + id: index, + title: item.name, + subtitle: item.artists, + active: index === 0, + })); + } + + async sendCustomAction(action: string): Promise { + const playback = this.playback; + if (!playback) return; + const client = this.deps.client; + if (action === "shuffle") { + const shuffle = !playback.shuffle; + await this.command(() => client.setShuffle(shuffle), () => { + playback.shuffle = shuffle; + // Shuffling reorders what comes next. + this.queueFetchedAtMs = 0; + }); + } else if (action === "repeat") { + const repeat = REPEAT_NEXT[playback.repeat]; + await this.command(() => client.setRepeat(repeat), () => { + playback.repeat = repeat; + }); + } else if (action === "like") { + const liked = this.liked; + if (!liked || liked.value === null) return; + const value = !liked.value; + await this.command(() => (value ? client.saveToLibrary([liked.uri]) : client.removeFromLibrary([liked.uri])), () => { + if (this.liked === liked) liked.value = value; + }); + } + } + + async refresh(): Promise { + await this.poll(); + } + + /** + * Play something (a browsed item, a search result) or, with an empty + * request, resume. With no active device, starts on one the account has + * open (preferring a phone). Resolves to whether it worked; a failure is + * left as the state's notice. + */ + async startPlayback(request: SpotifyPlayRequest): Promise { + const client = this.deps.client; + const playback = this.playback; + const ok = await this.command(async () => { + try { + await client.play(request); + } catch (error) { + if (!(error instanceof SpotifyApiError) || error.reason !== "NO_ACTIVE_DEVICE") throw error; + const device = pickDevice(await client.devices()); + if (!device) throw error; + await client.play(request, device.id); + } + }, () => { + this.lastPlayingAtMs = this.now(); + if (playback && this.playback === playback && !request.contextUri && !request.uris) { + playback.playing = true; + this.fetchedAtMs = this.now(); + } + }); + return ok; + } + + /** The account changed (signed in, out, or as someone else): forget the old account's state. */ + accountChanged(): void { + this.playback = null; + this.upcoming = []; + this.queueCurrentUri = ""; + this.art = null; + this.artVersion++; + this.liked = null; + this.lastPlayingAtMs = 0; + this.emit(true); + if (this.listeners.size) this.schedulePoll(0); + } + + /** Show a message about a failed command in the state's notice for a few seconds. */ + showNotice(message: string): void { + this.notice = message; + this.noticeUntilMs = this.now() + NOTICE_MS; + if (this.noticeTimer !== null) clearTimeout(this.noticeTimer); + this.noticeTimer = setTimeout(() => { + this.noticeTimer = null; + this.emit(true); + }, NOTICE_MS + 50); + this.emit(true); + } + + private currentNotice(): string { + return this.notice && this.now() < this.noticeUntilMs ? this.notice : ""; + } + + private customActions(playback: SpotifyPlayback): MediaCustomAction[] { + if (!playback.item) return []; + const actions: MediaCustomAction[] = []; + if (!playback.disallows.includes("toggling_shuffle")) { + actions.push({ action: "shuffle", name: playback.shuffle ? "Shuffle: on" : "Shuffle: off" }); + } + if (!playback.disallows.includes("toggling_repeat_context") || !playback.disallows.includes("toggling_repeat_track")) { + actions.push({ action: "repeat", name: REPEAT_LABEL[playback.repeat] }); + } + const liked = this.liked; + if (playback.item.type === "track" && liked?.uri === playback.item.uri && liked.value !== null) { + actions.push({ action: "like", name: liked.value ? "Unlike song" : "Like song" }); + } + return actions; + } + + /** Run a command; on success apply its expected effect at once, then re-read to confirm. */ + private async command(run: () => Promise, applied?: () => void): Promise { + let ok = false; + try { + await run(); + ok = true; + applied?.(); + this.emit(true); + } catch (error) { + this.showNotice(errorMessage(error)); + } + this.settlePolls = SETTLE_POLLS; + this.schedulePoll(SETTLE_POLL_MS); + return ok; + } + + private async sendVolume(deviceId: string): Promise { + this.volumeSending = true; + try { + while (this.volumeTarget !== null) { + const volume = this.volumeTarget; + try { + await this.deps.client.setVolume(volume, deviceId); + } catch (error) { + this.showNotice(errorMessage(error)); + } + if (this.volumeTarget === volume) this.volumeTarget = null; + } + } finally { + this.volumeSending = false; + } + } + + private poll(): Promise { + if (this.polling) { + this.pollAgain = true; + return this.polling; + } + this.polling = this.pollOnce().finally(() => { + this.polling = null; + if (this.pollAgain) { + this.pollAgain = false; + void this.poll(); + } + }); + return this.polling; + } + + private async pollOnce(): Promise { + this.clearPollTimer(); + if (!this.deps.isSignedIn()) { + if (this.playback) { + this.playback = null; + this.emit(true); + } + return; + } + let delay: number; + try { + const playback = await this.deps.client.playback(); + this.fetchedAtMs = this.now(); + this.applyPlayback(playback); + delay = this.nextPollDelay(playback); + } catch (error) { + if (!(error instanceof SpotifyApiError) || error.reason !== "signed_out") { + console.warn(`[spotify] playback read failed: ${error}`); + } + // Revoked access signs out (SpotifySession); the account listener resets the state. + delay = POLL_ERROR_MS; + } + if (this.settlePolls > 0) { + this.settlePolls--; + delay = Math.min(delay, 1_500); + } + this.emit(false); + this.schedulePoll(delay); + } + + private applyPlayback(playback: SpotifyPlayback | null): void { + const previousUri = this.playback?.item?.uri ?? ""; + this.playback = playback; + if (playback?.playing) this.lastPlayingAtMs = this.now(); + const item = playback?.item ?? null; + if ((item?.uri ?? "") !== previousUri) this.itemChanged(item); + } + + /** A new item: fetch its art and whether it is liked. */ + private itemChanged(item: SpotifyItem | null): void { + const url = item ? pickArtUrl(item) : ""; + if (url !== (this.art?.url ?? "")) { + this.artVersion++; + this.art = url ? { url, bytes: null, decoded: new Map() } : null; + if (url) void this.loadArt(url); + } + this.liked = item?.type === "track" ? { uri: item.uri, value: null } : null; + if (item?.type === "track") void this.loadLiked(item.uri); + } + + private async loadArt(url: string): Promise { + let bytes: Uint8Array | null = null; + try { + bytes = await this.deps.download(url); + } catch (error) { + console.warn(`[spotify] album art download failed: ${error}`); + } + if (!bytes?.length || this.art?.url !== url) return; + this.art.bytes = bytes; + this.artVersion++; + this.emit(true); + } + + private async loadLiked(uri: string): Promise { + try { + const [value] = await this.deps.client.libraryContains([uri]); + if (this.liked?.uri !== uri) return; + this.liked.value = Boolean(value); + this.emit(true); + } catch (error) { + console.warn(`[spotify] library check failed: ${error}`); + } + } + + private async loadQueue(): Promise { + this.queueLoading = true; + try { + const queue = await this.deps.client.queue(); + this.upcoming = queue.upcoming; + this.queueCurrentUri = queue.current?.uri ?? ""; + this.queueFetchedAtMs = this.now(); + this.emit(true); + } catch (error) { + console.warn(`[spotify] queue read failed: ${error}`); + this.queueFetchedAtMs = this.now(); + } finally { + this.queueLoading = false; + } + } + + private nextPollDelay(playback: SpotifyPlayback | null): number { + if (!playback) return POLL_IDLE_MS; + if (!playback.playing) return POLL_PAUSED_MS; + const durationMs = playback.item?.durationMs ?? -1; + // Catch the change to the next track as it happens. + const remaining = durationMs > 0 ? durationMs - playback.progressMs + 700 : POLL_PLAYING_MS; + return Math.max(1_000, Math.min(POLL_PLAYING_MS, remaining)); + } + + private schedulePoll(delayMs: number): void { + if (!this.listeners.size && !this.settlePolls) return; + this.clearPollTimer(); + this.pollTimer = setTimeout(() => { + this.pollTimer = null; + void this.poll(); + }, delayMs); + } + + private clearPollTimer(): void { + if (this.pollTimer !== null) clearTimeout(this.pollTimer); + this.pollTimer = null; + } + + /** Tell listeners; unforced, only when something they draw (other than the advancing position) changed. */ + private emit(force: boolean): void { + if (!this.listeners.size) return; + const state = this.snapshot(); + const signature = JSON.stringify({ ...state, positionMs: 0 }); + if (!force && signature === this.lastSignature) return; + this.lastSignature = signature; + for (const listener of this.listeners) listener(state); + } +} + +/** "Spotify", or "Spotify · " when it plays elsewhere than a phone. */ +export function spotifyAppName(device: SpotifyDevice | null): string { + if (!device?.name || device.type === "Smartphone") return SPOTIFY_APP_NAME; + return `${SPOTIFY_APP_NAME} · ${device.name}`; +} + +/** Spotify sets a speaker's or computer's volume; a phone's (this one) goes through the system volume. */ +function usesDeviceVolume(device: SpotifyDevice | null): boolean { + return Boolean(device && device.type !== "Smartphone" && device.supportsVolume && device.volumePercent >= 0 && device.id); +} + +/** A device to start playback on when none is active: an open one, a phone first. */ +export function pickDevice(devices: SpotifyDevice[]): SpotifyDevice | null { + const usable = devices.filter((device) => device.id && !device.restricted); + return usable.find((device) => device.active) + ?? usable.find((device) => device.type === "Smartphone") + ?? usable[0] + ?? null; +} + +function pickArtUrl(item: SpotifyItem): string { + const images = [...item.images].sort((a, b) => a.width - b.width); + return (images.find((image) => image.width >= ART_MIN_WIDTH) ?? images[images.length - 1])?.url ?? ""; +} + +export function errorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} diff --git a/app/native/spotify-service.ts b/app/native/spotify-service.ts new file mode 100644 index 00000000..320937ac --- /dev/null +++ b/app/native/spotify-service.ts @@ -0,0 +1,159 @@ +import type { GrayImage } from "../graphics/image"; +import { onSettingsStoreChanged } from "./settings-store"; +import { + readSpotifyAccount, + saveSpotifyAccount, + signInToSpotify, + SPOTIFY_ACCOUNT_KEY, + SPOTIFY_CLIENT_ID_KEY, + SpotifySession, + type SpotifyAccount, + type WebSignIn, +} from "./spotify-auth"; +import { SpotifyLibrary } from "./spotify-library"; +import { errorMessage, SpotifyPlayer } from "./spotify-player"; +import { SpotifyClient, type SpotifyFetch, type SpotifySearchType } from "./spotify-web-api"; + +/** Give up on a sign-in nobody finished on the phone. */ +const SIGN_IN_TIMEOUT_MS = 10 * 60_000; + +/** What the platform supplies (spotify.ios.ts). */ +export type SpotifyPlatform = { + fetch: SpotifyFetch; + webSignIn: WebSignIn; + download(url: string): Promise; + decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null; + systemVolume: { get(): number; set(percent: number): void }; +}; + +export type SpotifySignInState = + | { phase: "idle"; error: string } + /** The sign-in sheet is open on the phone, or waiting for Faceclaw to come to the foreground. */ + | { phase: "waiting"; error: "" }; + +/** + * Spotify for the Music app, the media bridges and the assistant: the + * account and its sign-in, the Web API client, the player and the library. + */ +export class SpotifyService { + readonly client: SpotifyClient; + readonly player: SpotifyPlayer; + readonly library: SpotifyLibrary; + private signInState: SpotifySignInState = { phase: "idle", error: "" }; + private readonly listeners = new Set<() => void>(); + private accountKey: string; + + constructor(private readonly platform: SpotifyPlatform) { + const isSignedIn = () => readSpotifyAccount() !== null; + this.client = new SpotifyClient(platform.fetch, new SpotifySession(platform.fetch)); + this.player = new SpotifyPlayer({ + client: this.client, + isSignedIn, + download: (url) => platform.download(url), + decodeArt: (bytes, maxSize) => platform.decodeArt(bytes, maxSize), + systemVolume: platform.systemVolume, + }); + this.library = new SpotifyLibrary({ + client: this.client, + player: this.player, + isSignedIn, + userId: () => readSpotifyAccount()?.userId ?? "", + }); + this.accountKey = accountIdentity(readSpotifyAccount()); + onSettingsStoreChanged((key) => { + if (key !== SPOTIFY_ACCOUNT_KEY && key !== SPOTIFY_CLIENT_ID_KEY) return; + // Token refreshes rewrite the account too; only a different sign-in resets the player. + const identity = accountIdentity(readSpotifyAccount()); + if (identity !== this.accountKey) { + this.accountKey = identity; + this.player.accountChanged(); + } + this.emit(); + }); + } + + account(): SpotifyAccount | null { + return readSpotifyAccount(); + } + + signInStatus(): SpotifySignInState { + return this.signInState; + } + + /** Repaint hook for the sign-in menu: account or sign-in progress changed. */ + onChange(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + /** + * Open Spotify's sign-in page on the phone (it waits for Faceclaw to be in + * the foreground); the outcome lands in signInStatus(). + */ + async signIn(): Promise { + if (this.signInState.phase === "waiting") return; + this.signInState = { phase: "waiting", error: "" }; + this.emit(); + const timer = setTimeout(() => this.platform.webSignIn.cancel(), SIGN_IN_TIMEOUT_MS); + try { + await signInToSpotify(this.platform.fetch, this.platform.webSignIn); + this.signInState = { phase: "idle", error: "" }; + } catch (error) { + const message = errorMessage(error); + this.signInState = { phase: "idle", error: message === "cancelled" ? "" : message }; + } finally { + clearTimeout(timer); + this.emit(); + } + } + + cancelSignIn(): void { + this.platform.webSignIn.cancel(); + } + + signOut(): void { + saveSpotifyAccount(null); + this.signInState = { phase: "idle", error: "" }; + this.emit(); + } + + /** + * Search Spotify and play the best match: a track, or the album, artist or + * playlist when the request names that kind. Resolves to what started, or + * rejects with why not. + */ + async playSearch(query: string, kind: SpotifySearchType = "track"): Promise { + if (!readSpotifyAccount()) throw new Error("Not signed in to Spotify. Sign in under Spotify in the Music app menu on the glasses."); + const results = await this.client.search(query, [kind], 3); + let request: { contextUri?: string; uris?: string[] } | null = null; + let description = ""; + if (kind === "track" && results.tracks[0]) { + const track = results.tracks[0]; + request = { uris: [track.uri] }; + description = `${track.name}${track.artists ? ` by ${track.artists}` : ""}`; + } else if (kind === "album" && results.albums[0]) { + const album = results.albums[0]; + request = { contextUri: album.uri }; + description = `the album ${album.name}${album.artists ? ` by ${album.artists}` : ""}`; + } else if (kind === "artist" && results.artists[0]) { + request = { contextUri: results.artists[0].uri }; + description = results.artists[0].name; + } else if (kind === "playlist" && results.playlists[0]) { + request = { contextUri: results.playlists[0].uri }; + description = `the playlist ${results.playlists[0].name}`; + } + if (!request) throw new Error(`Spotify found no ${kind} matching "${query}".`); + if (!(await this.player.startPlayback(request))) { + throw new Error(this.player.snapshot().notice || "Spotify didn't start playback."); + } + return description; + } + + private emit(): void { + for (const listener of this.listeners) listener(); + } +} + +function accountIdentity(account: SpotifyAccount | null): string { + return account ? `${account.clientId}/${account.userId}` : ""; +} diff --git a/app/native/spotify-web-api.ts b/app/native/spotify-web-api.ts new file mode 100644 index 00000000..82697c19 --- /dev/null +++ b/app/native/spotify-web-api.ts @@ -0,0 +1,474 @@ +/** + * Spotify Web API over HTTPS with the signed-in user's token: the player + * (what is playing on any of the user's Spotify Connect devices, and control + * of it), the user's library, and search, normalized into the small shapes + * the Music app uses. No NativeScript imports, so the Node tests drive it with + * a fake fetch. + * + * Every personal client ID is a development-mode app, which since February + * 2026 means: the client ID's owner needs Premium, playback control needs + * Premium, at most five allowlisted users, only owned or collaborative + * playlists list their items, no batch lookups, and search and artist albums + * return at most 10 results per page. + */ + +export const SPOTIFY_API_BASE = "https://api.spotify.com/v1"; + +const HTTP_TIMEOUT_MS = 12_000; +/** The most a library listing gathers (a page is 50); enough to scroll through on the glasses. */ +const MAX_LIST_ITEMS = 500; + +/** The subset of fetch this client needs (fetchTextWithUserAgent in the app). */ +export type SpotifyFetch = ( + url: string, + init: { method: string; headers: Record; body?: string }, +) => Promise<{ status: number; text(): Promise }>; + +/** Hands out a current access token, refreshing it when expired or when forced (after a 401). */ +export type SpotifyTokenSource = { + accessToken(forceRefresh: boolean): Promise; +}; + +/** A failed request; `reason` is Spotify's player error code (NO_ACTIVE_DEVICE, PREMIUM_REQUIRED, ...). */ +export class SpotifyApiError extends Error { + constructor( + message: string, + readonly status: number, + readonly reason = "", + ) { + super(message); + this.name = "SpotifyApiError"; + } +} + +export type SpotifyImage = { url: string; width: number; height: number }; + +export type SpotifyItem = { + uri: string; + type: "track" | "episode"; + name: string; + /** Track artists, comma-joined; for an episode, its show. */ + artists: string; + /** The track's album; for an episode, its show. "" when the API leaves it out (album track listings). */ + album: string; + durationMs: number; + images: SpotifyImage[]; +}; + +export type SpotifyDevice = { + id: string; + name: string; + /** "Smartphone", "Computer", "Speaker", ... */ + type: string; + active: boolean; + /** Restricted devices accept no Web API commands. */ + restricted: boolean; + /** -1 when the device reports none. */ + volumePercent: number; + supportsVolume: boolean; +}; + +export type SpotifyRepeat = "off" | "context" | "track"; + +export type SpotifyPlayback = { + device: SpotifyDevice | null; + playing: boolean; + progressMs: number; + shuffle: boolean; + repeat: SpotifyRepeat; + /** The playlist / album / artist playing from; "" for none (a queue of single tracks). */ + contextUri: string; + /** null during an ad, between items, or for content the API does not describe. */ + item: SpotifyItem | null; + /** "track", "episode", "ad" or "unknown". */ + playingType: string; + /** The player actions the context forbids, as Spotify names them: pausing, resuming, skipping_next, ... */ + disallows: string[]; +}; + +export type SpotifyQueue = { current: SpotifyItem | null; upcoming: SpotifyItem[] }; + +export type SpotifyUser = { id: string; displayName: string; product: string }; + +export type SpotifyPlaylist = { + id: string; + uri: string; + name: string; + ownerId: string; + ownerName: string; + collaborative: boolean; + total: number; +}; + +export type SpotifyAlbum = { id: string; uri: string; name: string; artists: string }; + +export type SpotifyArtist = { id: string; uri: string; name: string }; + +/** A played track and the context it played from ("" when none). */ +export type SpotifyPlayed = { item: SpotifyItem; contextUri: string }; + +export type SpotifySearchResults = { + tracks: SpotifyItem[]; + albums: SpotifyAlbum[]; + artists: SpotifyArtist[]; + playlists: SpotifyPlaylist[]; +}; + +export type SpotifySearchType = "track" | "album" | "artist" | "playlist"; + +/** + * What to play: a context (playlist, album, artist, show) optionally starting + * at one of its items, or a list of track URIs. Empty resumes the current + * playback. + */ +export type SpotifyPlayRequest = { + contextUri?: string; + uris?: string[]; + offsetUri?: string; + offsetPosition?: number; +}; + +type Query = Record; + +export class SpotifyClient { + constructor( + private readonly fetchImpl: SpotifyFetch, + private readonly tokens: SpotifyTokenSource, + ) {} + + /** The current playback on whichever device is active; null when nothing is. */ + async playback(): Promise { + const raw = await this.request("GET", "/me/player", { query: { additional_types: "episode" } }); + return raw ? parsePlayback(raw) : null; + } + + async queue(): Promise { + const raw = await this.request("GET", "/me/player/queue"); + return { + current: parseItem(raw?.currently_playing), + upcoming: asArray(raw?.queue).map(parseItem).filter(isPresent), + }; + } + + async devices(): Promise { + const raw = await this.request("GET", "/me/player/devices"); + return asArray(raw?.devices).map(parseDevice).filter(isPresent); + } + + async play(request: SpotifyPlayRequest = {}, deviceId = ""): Promise { + const body: Record = {}; + if (request.contextUri) body.context_uri = request.contextUri; + if (request.uris?.length) body.uris = request.uris; + if (request.offsetUri) body.offset = { uri: request.offsetUri }; + else if (request.offsetPosition !== undefined) body.offset = { position: request.offsetPosition }; + await this.request("PUT", "/me/player/play", { query: { device_id: deviceId || undefined }, body }); + } + + async pause(): Promise { + await this.request("PUT", "/me/player/pause"); + } + + async next(): Promise { + await this.request("POST", "/me/player/next"); + } + + async previous(): Promise { + await this.request("POST", "/me/player/previous"); + } + + async setShuffle(on: boolean): Promise { + await this.request("PUT", "/me/player/shuffle", { query: { state: on } }); + } + + async setRepeat(mode: SpotifyRepeat): Promise { + await this.request("PUT", "/me/player/repeat", { query: { state: mode } }); + } + + async setVolume(percent: number, deviceId = ""): Promise { + const volume = Math.max(0, Math.min(100, Math.round(percent))); + await this.request("PUT", "/me/player/volume", { query: { volume_percent: volume, device_id: deviceId || undefined } }); + } + + /** Move playback to a device (one that shows in devices() but is not active), optionally playing. */ + async transfer(deviceId: string, play: boolean): Promise { + await this.request("PUT", "/me/player", { body: { device_ids: [deviceId], play } }); + } + + async me(): Promise { + const raw = await this.request("GET", "/me"); + return { id: str(raw?.id), displayName: str(raw?.display_name) || str(raw?.id), product: str(raw?.product) }; + } + + /** Whether each URI is in the user's library (Liked Songs for tracks). */ + async libraryContains(uris: string[]): Promise { + const raw = await this.request("GET", "/me/library/contains", { query: { uris: uris.join(",") } }); + return asArray(raw).map(Boolean); + } + + async saveToLibrary(uris: string[]): Promise { + await this.request("PUT", "/me/library", { query: { uris: uris.join(",") } }); + } + + async removeFromLibrary(uris: string[]): Promise { + await this.request("DELETE", "/me/library", { query: { uris: uris.join(",") } }); + } + + playlists(): Promise { + return this.collect("/me/playlists", { limit: 50 }, (raw) => asArray(raw?.items).map(parsePlaylist)); + } + + /** Only playlists the user owns or collaborates on list their items (403 otherwise). */ + playlistItems(playlistId: string): Promise { + return this.collect(`/playlists/${encodeURIComponent(playlistId)}/items`, { limit: 50, additional_types: "track,episode" }, + (raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.item ?? entry?.track))); + } + + savedTracks(): Promise { + return this.collect("/me/tracks", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.track))); + } + + savedAlbums(): Promise { + return this.collect("/me/albums", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseAlbum(entry?.album))); + } + + albumTracks(albumId: string): Promise { + return this.collect(`/albums/${encodeURIComponent(albumId)}/tracks`, { limit: 50 }, (raw) => asArray(raw?.items).map(parseItem)); + } + + followedArtists(): Promise { + // Cursor-paged, with the page wrapped in an "artists" object. + return this.collect("/me/following", { type: "artist", limit: 50 }, (raw) => asArray(raw?.artists?.items).map(parseArtist), + (raw) => str(raw?.artists?.next)); + } + + artistAlbums(artistId: string): Promise { + return this.collect(`/artists/${encodeURIComponent(artistId)}/albums`, { include_groups: "album,single,compilation", limit: 10 }, + (raw) => asArray(raw?.items).map(parseAlbum), undefined, 100); + } + + /** Recently played tracks, newest first (one page: the API keeps only the last 50). */ + async recentlyPlayed(): Promise { + const raw = await this.request("GET", "/me/player/recently-played", { query: { limit: 50 } }); + return asArray(raw?.items) + .map((entry: any) => { + const item = parseItem(entry?.track); + return item ? { item, contextUri: str(entry?.context?.uri) } : null; + }) + .filter(isPresent); + } + + async search(query: string, types: SpotifySearchType[], limit = 5): Promise { + const raw = await this.request("GET", "/search", { + query: { q: query, type: types.join(","), limit: Math.max(1, Math.min(10, Math.round(limit))) }, + }); + return { + tracks: asArray(raw?.tracks?.items).map(parseItem).filter(isPresent), + albums: asArray(raw?.albums?.items).map(parseAlbum).filter(isPresent), + artists: asArray(raw?.artists?.items).map(parseArtist).filter(isPresent), + // Search results include nulls for playlists the API can no longer return. + playlists: asArray(raw?.playlists?.items).map(parsePlaylist).filter(isPresent), + }; + } + + /** Gather a paged listing by following its `next` links, up to maxItems. */ + private async collect( + path: string, + query: Query, + parse: (raw: any) => Array, + nextOf: (raw: any) => string = (raw) => str(raw?.next), + maxItems = MAX_LIST_ITEMS, + ): Promise { + const items: T[] = []; + let raw = await this.request("GET", path, { query }); + for (;;) { + for (const item of parse(raw)) if (item) items.push(item); + const next = nextOf(raw); + if (!next || items.length >= maxItems) return items.slice(0, maxItems); + raw = await this.request("GET", next); + } + } + + /** + * One API call. `path` is relative to the API base, or an absolute `next` + * link from a page. Resolves to the parsed JSON body (null when empty); + * a 401 refreshes the token and retries once. + */ + private async request(method: string, path: string, options: { query?: Query; body?: unknown } = {}): Promise { + const url = path.startsWith("https://") ? path : `${SPOTIFY_API_BASE}${path}${buildQuery(options.query)}`; + const headers: Record = { Accept: "application/json" }; + if (options.body !== undefined) headers["Content-Type"] = "application/json"; + // A command without a body still sends an empty one: Spotify answers a + // PUT or POST with no Content-Length with 411. + const body = options.body !== undefined ? JSON.stringify(options.body) : method === "GET" ? undefined : ""; + for (let attempt = 0; ; attempt++) { + const authorization = `Bearer ${await this.tokens.accessToken(attempt > 0)}`; + const response = await withTimeout(this.fetchImpl(url, { method, headers: { ...headers, Authorization: authorization }, body })); + const text = await withTimeout(response.text()); + if (response.status === 401 && attempt === 0) continue; + if (response.status >= 200 && response.status < 300) { + if (!text.trim()) return null; + try { + return JSON.parse(text); + } catch { + // Commands answer 200 with a snapshot id or plain text; nothing to read. + return null; + } + } + throw apiError(response.status, text); + } + } +} + +/** Form / query encoding (not URLSearchParams, which the NativeScript runtime mishandles). */ +export function buildQuery(query: Query | undefined): string { + if (!query) return ""; + const parts = Object.keys(query) + .filter((key) => query[key] !== undefined && query[key] !== "") + .map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(String(query[key]))}`); + return parts.length ? `?${parts.join("&")}` : ""; +} + +async function withTimeout(promise: Promise): Promise { + let timer: ReturnType | null = null; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new SpotifyApiError("Spotify didn't answer. Check the phone's connection.", 0)), HTTP_TIMEOUT_MS); + }); + try { + return await Promise.race([promise, timeout]); + } catch (error) { + if (error instanceof SpotifyApiError) throw error; + throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0); + } finally { + if (timer) clearTimeout(timer); + } +} + +/** A message for the glasses from Spotify's error body ({error: {status, message, reason}}). */ +function apiError(status: number, body: string): SpotifyApiError { + let message = ""; + let reason = ""; + try { + const parsed = JSON.parse(body); + message = str(parsed?.error?.message ?? parsed?.error_description); + reason = str(parsed?.error?.reason); + } catch { + // Not JSON (a gateway error page). + } + switch (reason) { + case "NO_ACTIVE_DEVICE": + return new SpotifyApiError("Spotify isn't open on any device. Open Spotify on your phone, then try again.", status, reason); + case "PREMIUM_REQUIRED": + return new SpotifyApiError("Controlling Spotify playback needs Spotify Premium.", status, reason); + } + if (status === 401) return new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, reason); + if (status === 403 && /registered|developer dashboard/i.test(message)) { + return new SpotifyApiError("This Spotify account isn't on the client ID's user list (Developer Dashboard > User Management).", status, reason); + } + if (status === 429) return new SpotifyApiError("Spotify is limiting requests; try again in a minute.", status, reason); + return new SpotifyApiError(message ? `Spotify: ${message}` : `Spotify error (HTTP ${status}).`, status, reason); +} + +export function parsePlayback(raw: any): SpotifyPlayback { + const repeat = str(raw?.repeat_state); + const disallows = raw?.actions?.disallows; + return { + device: parseDevice(raw?.device), + playing: raw?.is_playing === true, + progressMs: num(raw?.progress_ms, 0), + shuffle: raw?.shuffle_state === true, + repeat: repeat === "context" || repeat === "track" ? repeat : "off", + contextUri: str(raw?.context?.uri), + item: parseItem(raw?.item), + playingType: str(raw?.currently_playing_type) || "unknown", + disallows: disallows && typeof disallows === "object" ? Object.keys(disallows).filter((key) => disallows[key] === true) : [], + }; +} + +export function parseItem(raw: any): SpotifyItem | null { + if (!raw || typeof raw !== "object" || typeof raw.uri !== "string" || !raw.uri) return null; + if (raw.type === "episode") { + const show = str(raw.show?.name); + return { + uri: raw.uri, + type: "episode", + name: str(raw.name), + artists: show || str(raw.show?.publisher), + album: show, + durationMs: num(raw.duration_ms, -1), + images: parseImages(raw.images?.length ? raw.images : raw.show?.images), + }; + } + return { + uri: raw.uri, + type: "track", + name: str(raw.name), + artists: joinNames(raw.artists), + album: str(raw.album?.name), + durationMs: num(raw.duration_ms, -1), + images: parseImages(raw.album?.images), + }; +} + +export function parseDevice(raw: any): SpotifyDevice | null { + if (!raw || typeof raw !== "object") return null; + const volume = raw.volume_percent; + return { + id: str(raw.id), + name: str(raw.name), + type: str(raw.type), + active: raw.is_active === true, + restricted: raw.is_restricted === true, + volumePercent: typeof volume === "number" && Number.isFinite(volume) ? volume : -1, + supportsVolume: raw.supports_volume === true, + }; +} + +function parsePlaylist(raw: any): SpotifyPlaylist | null { + if (!raw || typeof raw !== "object" || !raw.id) return null; + return { + id: str(raw.id), + uri: str(raw.uri) || `spotify:playlist:${raw.id}`, + name: str(raw.name), + ownerId: str(raw.owner?.id), + ownerName: str(raw.owner?.display_name) || str(raw.owner?.id), + collaborative: raw.collaborative === true, + // `items` replaced the deprecated `tracks` summary in 2026. + total: num(raw.items?.total ?? raw.tracks?.total, -1), + }; +} + +function parseAlbum(raw: any): SpotifyAlbum | null { + if (!raw || typeof raw !== "object" || !raw.id) return null; + return { id: str(raw.id), uri: str(raw.uri) || `spotify:album:${raw.id}`, name: str(raw.name), artists: joinNames(raw.artists) }; +} + +function parseArtist(raw: any): SpotifyArtist | null { + if (!raw || typeof raw !== "object" || !raw.id) return null; + return { id: str(raw.id), uri: str(raw.uri) || `spotify:artist:${raw.id}`, name: str(raw.name) }; +} + +function parseImages(raw: any): SpotifyImage[] { + return asArray(raw) + .filter((image: any) => image && typeof image.url === "string" && image.url) + .map((image: any) => ({ url: image.url, width: num(image.width, 0), height: num(image.height, 0) })); +} + +function joinNames(raw: any): string { + return asArray(raw).map((entry: any) => str(entry?.name)).filter(Boolean).join(", "); +} + +function asArray(value: unknown): any[] { + return Array.isArray(value) ? value : []; +} + +function str(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function num(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function isPresent(value: T | null | undefined): value is T { + return value !== null && value !== undefined; +} diff --git a/app/native/spotify.ios.ts b/app/native/spotify.ios.ts new file mode 100644 index 00000000..43de82e2 --- /dev/null +++ b/app/native/spotify.ios.ts @@ -0,0 +1,63 @@ +import { fetchTextWithUserAgent } from "../util/http"; +import { downloadBinary } from "./binary-http.ios"; +import { grayImageFromPacket, PHOTO_TONE } from "./image-files"; +import { fromData, toData } from "./kotlin-data"; +import type { WebSignIn } from "./spotify-auth"; +import { SpotifyService } from "./spotify-service"; + +declare const FaceclawAppleMusic: any; +declare const FaceclawCrypto: any; +declare const FaceclawGraphics: any; +declare const FaceclawKitIosProtocol: any; +declare const FaceclawWebSignIn: any; + +const ART_DOWNLOAD_TIMEOUT_MS = 10_000; + +/** Sign-in in the system browser sheet, which FaceclawWebSignIn closes once the loopback redirect arrives. */ +const webSignIn: WebSignIn = { + createPkce: () => { + // 64 URL-safe characters (RFC 7636 allows 43..128). + const verifier = String(FaceclawCrypto.randomBase64Url(48)); + return { verifier, challenge: String(FaceclawCrypto.sha256Base64Url(verifier)) }; + }, + randomState: () => String(FaceclawCrypto.randomBase64Url(16)), + open: (authorizeUrl, port, path) => + new Promise((resolve, reject) => { + FaceclawWebSignIn.shared().signInPortPathCompletion(authorizeUrl, port, path, (redirectUrl: string | null, error: string | null) => { + if (redirectUrl) resolve(String(redirectUrl)); + else reject(new Error(String(error || "Sign-in failed"))); + }); + }), + cancel: () => FaceclawWebSignIn.shared().cancel(), +}; + +let protocol: any = null; +let service: SpotifyService | null = null; + +/** The Spotify service, created on first use (main thread only: it owns timers and the sign-in sheet). */ +export function spotifyService(): SpotifyService | null { + service ??= new SpotifyService({ + fetch: (url, init) => fetchTextWithUserAgent(url, init), + webSignIn, + download: async (url) => { + const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS); + return response.status === 200 ? response.bytes : null; + }, + decodeArt: (bytes, maxSize) => { + const packet = FaceclawGraphics.bgraPacketFromImageDataMaxSize(toData(bytes), Math.round(maxSize)); + if (!packet) return null; + // The shared Kotlin tone curve and dither, as for Apple Music's art. + protocol ??= FaceclawKitIosProtocol.new(); + return grayImageFromPacket(fromData(protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither))); + }, + // FaceclawAppleMusic's system output volume needs no media-library access. + systemVolume: { + get: () => { + const volume = Number(FaceclawAppleMusic.shared().volumePercent()); + return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1; + }, + set: (percent) => FaceclawAppleMusic.shared().setVolumePercent(Math.max(0, Math.min(100, Math.round(percent)))), + }, + }); + return service; +} diff --git a/app/native/spotify.ts b/app/native/spotify.ts new file mode 100644 index 00000000..28c32003 --- /dev/null +++ b/app/native/spotify.ts @@ -0,0 +1,9 @@ +import type { SpotifyService } from "./spotify-service"; + +/** + * Spotify through its Web API is wired up on iOS only (spotify.ios.ts): on + * Android, Spotify publishes a media session the Music app already controls. + */ +export function spotifyService(): SpotifyService | null { + return null; +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 87450ba7..916cadae 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -18,6 +18,7 @@ import { } from "~/assistant/models"; import { isLocalModelReady } from "../native/llama"; import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; +import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed"; import { MENU_ANIMATION_KEY } from "./menu-animation-pref"; @@ -1101,6 +1102,21 @@ export const mapboxApiKeySetting = new ConfigSettingString({ description: "Mapbox public token (pk. prefix), used by the Navigate app for maps, geocoding, and directions.", }); +/** Edited from the Music app's Spotify menu (iOS); a sign-in is tied to the client ID it was made with. */ +export const spotifyClientIdSetting = new ConfigSettingString({ + id: "spotify-client-id", + label: "Client ID", + storageKey: SPOTIFY_CLIENT_ID_KEY, + defaultValue: "", + editorTitle: "Spotify client ID (from your app in the Spotify Developer Dashboard)", + glassesEditTitle: "Edit Spotify client ID", + normalize: (value) => (value ?? "").trim(), + validate: (value) => (!value || isValidSpotifyClientId(value) ? null + : "A Spotify client ID is 32 letters and digits, shown on your app's page in the Spotify Developer Dashboard."), + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), + description: "The client ID of your own Spotify developer app, which Faceclaw signs in with.", +}); + /** * Staging buffer for the Terminal app's "Add connection" flow: the worker * asks the shell to open the phone text editor on this setting, the user diff --git a/tests/ios-apple-music.test.cjs b/tests/ios-apple-music.test.cjs index 35a0ebd5..297c3012 100644 --- a/tests/ios-apple-music.test.cjs +++ b/tests/ios-apple-music.test.cjs @@ -56,6 +56,8 @@ function controller(music = fakeMusic()) { './image-files': { grayImageFromPacket: bytes => (bytes && bytes.length ? { bytes } : null), PHOTO_TONE: { gamma: 2.2, dither: true } }, './kotlin-data': { fromData: data => data }, './media-types': types, + // Signed out of Spotify: the composite bridge follows Apple Music alone. + './spotify.ios': { spotifyService: () => null }, }, { FaceclawAppleMusic: { shared: () => music }, FaceclawKitIosProtocol: { new: () => ({ grayPacketBgraGammaDither: (packet, gamma, dither) => { @@ -155,7 +157,11 @@ test('access requests share one prompt, and denied or restricted access points t }); function browser(music = fakeMusic()) { - const api = load('app/native/media-browser.ios.ts', { './media-types': types }, { + const api = load('app/native/media-browser.ios.ts', { + './media-types': types, + './spotify.ios': { spotifyService: () => null }, + './spotify-player': { SPOTIFY_PACKAGE: 'com.spotify.client' }, + }, { FaceclawAppleMusic: { shared: () => music }, }); return { bridge: api.mediaBrowserBridge, music }; @@ -165,7 +171,7 @@ test('the Apple Music library browses only with access and drops answers after d const f = browser(); const [app] = f.bridge.listBrowsableApps(); assert.equal(app.appName, 'Apple Music'); - await assert.rejects(f.bridge.connect({ ...app, packageName: 'other' }), /Only Apple Music/); + await assert.rejects(f.bridge.connect({ ...app, packageName: 'other' }), /Not an Apple Music library/); await assert.rejects(f.bridge.browse('root'), /Not connected/); assert.equal(await f.bridge.connect(app), 'root'); diff --git a/tests/ios-calendar.test.cjs b/tests/ios-calendar.test.cjs index 07e92ee3..b85b0554 100644 --- a/tests/ios-calendar.test.cjs +++ b/tests/ios-calendar.test.cjs @@ -229,6 +229,7 @@ test('iOS assistant calendar tool awaits events, reports denied access and failu const { registerSystemTools } = load('app/assistant/system-tools.ts', { '../native/calendar': f.api, '../native/calendar-permissions': f.permissions, '../native/media-controller': { mediaControllerBridge: { start: async () => {} } }, + '../native/spotify': { spotifyService: () => null }, '../native/notification-icons': {}, '../ui/shell/shell': {}, './tool-registry': {}, }, { global: { isIOS: true } }); registerSystemTools({ registerSystemTool: (definition, handler) => registered.set(definition.name, handler) }); diff --git a/tests/ios-loopback-redirect-native.cjs b/tests/ios-loopback-redirect-native.cjs new file mode 100644 index 00000000..4236d61e --- /dev/null +++ b/tests/ios-loopback-redirect-native.cjs @@ -0,0 +1,18 @@ +// macOS integration test for the loopback OAuth redirect receiver compiled into iOS +// (Spotify sign-in): builds FaceclawLoopbackRedirect.m with a probe and talks HTTP to it. +const { execFileSync } = require('node:child_process'); +const { mkdtempSync, rmSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join, resolve } = require('node:path'); +const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-loopback-')); +try { + execFileSync('xcrun', ['clang', '-fobjc-arc', '-framework', 'Foundation', '-I', 'App_Resources/iOS/src', + 'tests/native/ios-loopback-redirect-test.m', 'App_Resources/iOS/src/FaceclawLoopbackRedirect.m', + '-o', join(dir, 'loopback-test')], { cwd: root, stdio: 'inherit' }); + // A port unlikely to be taken on a dev machine; the app's own port is in spotify-auth.ts. + execFileSync(join(dir, 'loopback-test'), ['43899'], { stdio: 'inherit' }); +} catch { + process.exitCode = 1; +} finally { + rmSync(dir, { recursive: true, force: true }); +} diff --git a/tests/ios-spotify.test.cjs b/tests/ios-spotify.test.cjs new file mode 100644 index 00000000..e4cfc5f3 --- /dev/null +++ b/tests/ios-spotify.test.cjs @@ -0,0 +1,530 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, modules = {}, globals = {}) { + const sandbox = { exports: {}, console, JSON, Promise, Date, Math, setTimeout, clearTimeout, require: id => { + if (!(id in modules)) throw new Error(`Unstubbed import: ${id}`); + return modules[id]; + }, ...globals }; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, sandbox); + return sandbox.exports; +} + +const CLIENT_ID = '0123456789abcdef0123456789abcdef'; + +/** Values from the sandbox carry its own Array/Object prototypes; compare their plain JSON form. */ +const plain = value => JSON.parse(JSON.stringify(value)); + +/** settings-store stand-in: synchronous change notifications. */ +function fakeSettings(initial = {}) { + const values = new Map(Object.entries(initial)); + const listeners = new Set(); + return { + values, + getStringSetting: (key, fallback) => (values.has(key) ? values.get(key) : fallback), + setStringSetting: (key, value) => { values.set(key, value); for (const listener of listeners) listener(key); }, + onSettingsStoreChanged: listener => { listeners.add(listener); return () => listeners.delete(listener); }, + }; +} + +/** Manual timers: run() fires everything due, in order. */ +function fakeTimers() { + let now = 1_000_000; let nextId = 1; + const pending = new Map(); + return { + now: () => now, + setTimeout: (fn, delay) => { const id = nextId++; pending.set(id, { fn, at: now + (delay || 0) }); return id; }, + clearTimeout: id => pending.delete(id), + advance(ms) { + now += ms; + for (const [id, timer] of [...pending].sort((a, b) => a[1].at - b[1].at)) { + if (timer.at <= now && pending.has(id)) { pending.delete(id); timer.fn(); } + } + }, + delays: () => [...pending.values()].map(timer => timer.at - now), + }; +} + +/** + * Spotify's endpoints as a route table: "METHOD /path" -> response or function(call). + * Records every call with its parsed URL, headers and body. + */ +function fakeFetch(routes) { + const calls = []; + const fetch = async (url, init) => { + const parsed = new URL(url); + const call = { url, method: init.method, path: parsed.pathname.replace('/v1', ''), query: Object.fromEntries(parsed.searchParams), + headers: init.headers, body: init.body }; + calls.push(call); + let route = routes[`${init.method} ${call.path}`]; + if (typeof route === 'function') route = route(call); + if (Array.isArray(route)) route = route.length > 1 ? route.shift() : route[0]; + const { status = 200, body = '' } = route ?? { status: 404, body: '{"error":{"status":404,"message":"no route"}}' }; + return { status, text: async () => (typeof body === 'string' ? body : JSON.stringify(body)) }; + }; + return { fetch, calls }; +} + +function loadSpotify({ settings = fakeSettings(), timers = null } = {}) { + const globals = timers ? { setTimeout: timers.setTimeout, clearTimeout: timers.clearTimeout } : {}; + const types = load('app/native/media-types.ts'); + const api = load('app/native/spotify-web-api.ts', {}, globals); + const auth = load('app/native/spotify-auth.ts', { './settings-store': settings, './spotify-web-api': api }, globals); + const player = load('app/native/spotify-player.ts', { './media-types': types, './spotify-web-api': api }, globals); + const library = load('app/native/spotify-library.ts', { './spotify-player': player }, globals); + const service = load('app/native/spotify-service.ts', { + './settings-store': settings, './spotify-auth': auth, './spotify-library': library, './spotify-player': player, './spotify-web-api': api, + }, globals); + return { types, api, auth, player, library, service, settings }; +} + +function signedIn(extra = {}) { + return fakeSettings({ + 'music.spotify.clientId': CLIENT_ID, + 'music.spotify.account': JSON.stringify({ clientId: CLIENT_ID, accessToken: 'access-1', refreshToken: 'refresh-1', + expiresAtMs: 9e15, userId: 'jim', displayName: 'Jim', product: 'premium', ...extra }), + }); +} + +const tokens = { accessToken: async force => (force ? 'fresh' : 'stale') }; + +function track(id, extra = {}) { + return { type: 'track', uri: `spotify:track:${id}`, name: `Song ${id}`, duration_ms: 200_000, + artists: [{ name: 'Band' }, { name: 'Guest' }], album: { name: 'Record', images: [ + { url: 'https://i.scdn.co/640', width: 640, height: 640 }, + { url: 'https://i.scdn.co/300', width: 300, height: 300 }, + { url: 'https://i.scdn.co/64', width: 64, height: 64 }] }, ...extra }; +} + +function playbackBody(extra = {}) { + return { device: { id: 'phone', name: "Jim's iPhone", type: 'Smartphone', is_active: true, volume_percent: 70, supports_volume: false }, + is_playing: true, progress_ms: 30_000, shuffle_state: false, repeat_state: 'off', context: { uri: 'spotify:playlist:p1' }, + item: track('a'), currently_playing_type: 'track', actions: { disallows: { resuming: true } }, ...extra }; +} + +test('the Web API client parses playback, retries a 401 with a fresh token and names player errors', async () => { + const { api } = loadSpotify(); + const f = fakeFetch({ + 'GET /me/player': [{ status: 401, body: '' }, { body: playbackBody({ + item: { type: 'episode', uri: 'spotify:episode:e', name: 'Ep', duration_ms: 1000, images: [], show: { name: 'Show', images: [{ url: 'u', width: 300 }] } }, + }) }], + 'PUT /me/player/play': { status: 404, body: { error: { status: 404, message: 'Player command failed: No active device found', reason: 'NO_ACTIVE_DEVICE' } } }, + 'POST /me/player/next': { status: 403, body: { error: { status: 403, message: 'Premium required', reason: 'PREMIUM_REQUIRED' } } }, + 'PUT /me/player/pause': { status: 204 }, + }); + const client = new api.SpotifyClient(f.fetch, tokens); + const playback = await client.playback(); + assert.deepEqual(f.calls.map(call => call.headers.Authorization), ['Bearer stale', 'Bearer fresh']); + assert.equal(f.calls[0].query.additional_types, 'episode'); + assert.equal(playback.playing, true); assert.equal(playback.device.type, 'Smartphone'); assert.equal(playback.device.volumePercent, 70); + assert.deepEqual(plain(playback.disallows), ['resuming']); assert.equal(playback.contextUri, 'spotify:playlist:p1'); + assert.equal(playback.item.type, 'episode'); assert.equal(playback.item.artists, 'Show'); assert.equal(playback.item.images[0].url, 'u'); + + await assert.rejects(client.play({ contextUri: 'spotify:album:x', offsetUri: 'spotify:track:y' }), + error => error instanceof api.SpotifyApiError && error.reason === 'NO_ACTIVE_DEVICE' && /Open Spotify/.test(error.message)); + const play = f.calls.find(call => call.path === '/me/player/play'); + assert.deepEqual(JSON.parse(play.body), { context_uri: 'spotify:album:x', offset: { uri: 'spotify:track:y' } }); + assert.equal(play.headers['Content-Type'], 'application/json'); + await assert.rejects(client.next(), /needs Spotify Premium/); + // Body-less commands still send an (empty) body, or Spotify answers 411. + await client.pause(); + assert.equal(f.calls.at(-1).body, ''); + assert.equal(f.calls.find(call => call.method === 'GET').body, undefined); +}); + +test('the Web API client follows next links and reads the renamed playlist item field', async () => { + const { api } = loadSpotify(); + const f = fakeFetch({ + 'GET /playlists/p1/items': call => (call.query.offset === '2' + ? { body: { items: [{ track: track('c') }], next: null } } + : { body: { items: [{ item: track('a') }, { item: null }, { item: track('b') }], next: 'https://api.spotify.com/v1/playlists/p1/items?offset=2' } }), + 'GET /me/following': { body: { artists: { items: [{ id: 'ar', name: 'Artist', uri: 'spotify:artist:ar' }], next: null } } }, + 'GET /search': { body: { tracks: { items: [track('s')] }, playlists: { items: [null, { id: 'pl', name: 'Mix', owner: { id: 'o' } }] } } }, + }); + const client = new api.SpotifyClient(f.fetch, tokens); + assert.deepEqual(plain((await client.playlistItems('p1')).map(item => item.name)), ['Song a', 'Song b', 'Song c']); + assert.equal(f.calls[0].query.limit, '50'); + assert.equal((await client.followedArtists())[0].name, 'Artist'); + const results = await client.search('daft punk', ['track', 'playlist'], 50); + assert.equal(f.calls.at(-1).query.limit, '10'); // development-mode maximum + assert.equal(f.calls.at(-1).query.q, 'daft punk'); + assert.equal(results.tracks[0].artists, 'Band, Guest'); + assert.equal(results.playlists.length, 1); assert.equal(results.playlists[0].uri, 'spotify:playlist:pl'); +}); + +test('sign-in builds a PKCE authorize URL, exchanges the code and stores the account', async () => { + const settings = fakeSettings({ 'music.spotify.clientId': CLIENT_ID }); + const { auth } = loadSpotify({ settings }); + const f = fakeFetch({ + 'POST /api/token': { body: { access_token: 'access-1', refresh_token: 'refresh-1', expires_in: 3600 } }, + 'GET /me': { body: { id: 'jim', display_name: 'Jim', product: 'premium' } }, + }); + let opened = null; + const webSignIn = { + createPkce: () => ({ verifier: 'verifier-1', challenge: 'challenge-1' }), + randomState: () => 'state-1', + open: async (url, port, redirectPath) => { + opened = { url, port, redirectPath }; + return 'http://127.0.0.1:43823/spotify-callback?code=AQB%2Fcode&state=state-1'; + }, + cancel: () => {}, + }; + const account = await auth.signInToSpotify(f.fetch, webSignIn, () => 5_000); + const authorize = new URL(opened.url); + assert.equal(authorize.origin + authorize.pathname, 'https://accounts.spotify.com/authorize'); + assert.equal(authorize.searchParams.get('client_id'), CLIENT_ID); + assert.equal(authorize.searchParams.get('redirect_uri'), 'http://127.0.0.1:43823/spotify-callback'); + assert.equal(authorize.searchParams.get('code_challenge_method'), 'S256'); + assert.equal(authorize.searchParams.get('code_challenge'), 'challenge-1'); + assert.match(authorize.searchParams.get('scope'), /user-modify-playback-state .*user-library-modify/); + assert.deepEqual([opened.port, opened.redirectPath], [43823, '/spotify-callback']); + + const exchange = Object.fromEntries(new URLSearchParams(f.calls[0].body)); + assert.deepEqual(exchange, { grant_type: 'authorization_code', code: 'AQB/code', redirect_uri: 'http://127.0.0.1:43823/spotify-callback', + client_id: CLIENT_ID, code_verifier: 'verifier-1' }); + assert.equal(f.calls[1].headers.Authorization, 'Bearer access-1'); + assert.equal(account.displayName, 'Jim'); assert.equal(account.expiresAtMs, 5_000 + 3_600_000); + assert.equal(auth.readSpotifyAccount().refreshToken, 'refresh-1'); + + // Another client ID: the stored sign-in no longer counts. + settings.setStringSetting('music.spotify.clientId', 'f'.repeat(32)); + assert.equal(auth.readSpotifyAccount(), null); +}); + +test('redirect parsing rejects mismatched state and reports a declined sign-in', () => { + const { auth } = loadSpotify(); + assert.equal(auth.parseSpotifyRedirect('http://127.0.0.1:1/cb?state=s&code=a%20b', 's'), 'a b'); + assert.throws(() => auth.parseSpotifyRedirect('http://127.0.0.1:1/cb?state=x&code=a', 's'), /didn't match/); + assert.throws(() => auth.parseSpotifyRedirect('http://127.0.0.1:1/cb?error=access_denied&state=s', 's'), /declined/); + assert.throws(() => auth.parseSpotifyRedirect('http://127.0.0.1:1/cb?state=s', 's'), /no authorization code/); +}); + +test('the session refreshes an expired token once for concurrent callers and signs out when revoked', async () => { + const settings = signedIn({ expiresAtMs: 0 }); + const { auth } = loadSpotify({ settings }); + let refreshes = 0; + const f = fakeFetch({ 'POST /api/token': () => { + refreshes++; + return refreshes === 1 ? { body: { access_token: 'access-2', refresh_token: 'refresh-2', expires_in: 3600 } } + : { status: 400, body: { error: 'invalid_grant', error_description: 'Refresh token revoked' } }; + } }); + const session = new auth.SpotifySession(f.fetch, () => 10_000); + assert.deepEqual(await Promise.all([session.accessToken(false), session.accessToken(false)]), ['access-2', 'access-2']); + assert.equal(refreshes, 1); + assert.equal(Object.fromEntries(new URLSearchParams(f.calls[0].body)).refresh_token, 'refresh-1'); + // The rotated refresh token is kept; the new access token is used until it nears expiry. + assert.equal(auth.readSpotifyAccount().refreshToken, 'refresh-2'); + assert.equal(await session.accessToken(false), 'access-2'); assert.equal(refreshes, 1); + + await assert.rejects(session.accessToken(true), /Sign in again/); + assert.equal(auth.readSpotifyAccount(), null); + await assert.rejects(session.accessToken(false), /Not signed in/); +}); + +function playerFixture({ routes = {}, settings = signedIn() } = {}) { + const timers = fakeTimers(); + const s = loadSpotify({ settings, timers }); + const f = fakeFetch({ 'GET /me/player': { body: playbackBody() }, 'GET /me/library/contains': { body: [false] }, + 'GET /me/player/queue': { body: { currently_playing: track('a'), queue: [track('b'), track('c', { artists: [{ name: 'Other' }] })] } }, + ...routes }); + const downloads = []; const decodes = []; const system = { volume: 40, set: [] }; + const service = new s.service.SpotifyService({ + fetch: f.fetch, + webSignIn: { createPkce: () => ({}), randomState: () => '', open: async () => '', cancel: () => {} }, + download: async url => { downloads.push(url); return new Uint8Array([1, 2, 3]); }, + decodeArt: (_bytes, size) => { decodes.push(size); return { size }; }, + systemVolume: { get: () => system.volume, set: value => system.set.push(value) }, + }); + // The player reads time from Date.now by default; give it the fake clock. + const player = new s.player.SpotifyPlayer({ client: service.client, isSignedIn: () => s.auth.readSpotifyAccount() !== null, + download: async url => { downloads.push(url); return new Uint8Array([1, 2, 3]); }, + decodeArt: (_bytes, size) => { decodes.push(size); return { size }; }, + systemVolume: { get: () => system.volume, set: value => system.set.push(value) }, now: timers.now }); + const flush = () => new Promise(resolve => setImmediate(resolve)); + const settle = async () => { for (let i = 0; i < 6; i++) await flush(); }; + return { ...s, f, timers, service, player, downloads, decodes, system, settle }; +} + +test('the player polls while listened to and maps playback, art and actions into the media state', async () => { + const p = playerFixture(); + const states = []; + const unsubscribe = p.player.onStateChange(state => states.push(state)); + assert.equal(states[0].available, false); assert.equal(states[0].accessEnabled, true); + p.timers.advance(0); await p.settle(); + const state = p.player.snapshot(); + assert.equal(state.available, true); assert.equal(state.playbackState, 'playing'); + assert.equal(state.title, 'Song a'); assert.equal(state.artist, 'Band, Guest'); assert.equal(state.album, 'Record'); + assert.equal(state.appName, 'Spotify'); assert.equal(state.packageName, 'com.spotify.client'); + assert.equal(state.canPlayPause, true); assert.equal(state.canSkipNext, true); + assert.deepEqual(plain(state.customActions.map(action => action.name)), ['Shuffle: off', 'Repeat: off', 'Like song']); + // The 300px image (the smallest at least 128 wide), downloaded once and decoded per size. + assert.deepEqual(p.downloads, ['https://i.scdn.co/300']); + assert.deepEqual(plain(p.player.getAlbumArt(112)), { size: 112 }); p.player.getAlbumArt(112); + assert.deepEqual(p.decodes, [112]); + // Position advances between polls. + p.timers.advance(1_000); + assert.equal(p.player.snapshot().positionMs, 31_000); + // Polling continues on a playing interval, and stops with the last listener. + assert.ok(p.timers.delays().some(delay => delay > 0 && delay <= 5_000)); + unsubscribe(); + assert.deepEqual(p.timers.delays(), []); +}); + +test('pausing applies at once, resuming with no active device starts on an open phone, and failures become a notice', async () => { + let active = true; + const p = playerFixture({ routes: { + 'PUT /me/player/pause': { status: 204 }, + 'PUT /me/player/play': call => (call.query.device_id || active ? { status: 204 } + : { status: 404, body: { error: { status: 404, reason: 'NO_ACTIVE_DEVICE', message: 'none' } } }), + 'GET /me/player/devices': { body: { devices: [ + { id: 'mac', name: 'MacBook', type: 'Computer', is_active: false }, + { id: 'iphone', name: 'iPhone', type: 'Smartphone', is_active: false }, + { id: 'tv', name: 'TV', type: 'TV', is_restricted: true }] } }, + 'POST /me/player/next': { status: 403, body: { error: { status: 403, reason: 'PREMIUM_REQUIRED', message: 'x' } } }, + } }); + p.player.onStateChange(() => {}); + p.timers.advance(0); await p.settle(); + await p.player.playPause(); + assert.equal(p.player.snapshot().playbackState, 'paused'); + assert.equal(p.f.calls.filter(call => call.path === '/me/player/pause').length, 1); + + active = false; + await p.player.playPause(); + const plays = p.f.calls.filter(call => call.path === '/me/player/play'); + assert.deepEqual(plays.map(call => call.query.device_id ?? ''), ['', 'iphone']); + assert.deepEqual(JSON.parse(plays[1].body), {}); + + await p.player.skipNext(); + assert.match(p.player.snapshot().notice, /Premium/); + // A settle poll re-reads soon after each command. + assert.ok(p.timers.delays().some(delay => delay <= 600)); + p.timers.advance(8_100); + assert.equal(p.player.snapshot().notice, ''); +}); + +test('the queue lists the current track first and jumping skips forward to the chosen item', async () => { + const p = playerFixture({ routes: { 'POST /me/player/next': { status: 204 } } }); + p.player.onStateChange(() => {}); + p.timers.advance(0); await p.settle(); + assert.deepEqual(plain(p.player.getQueue()), []); // first read in flight + await p.settle(); + assert.deepEqual(plain(p.player.getQueue().map(item => [item.id, item.title, item.subtitle, item.active])), + [[0, 'Song a', 'Band, Guest', true], [1, 'Song b', 'Band, Guest', false], [2, 'Song c', 'Other', false]]); + await p.player.skipToQueueItem(2); + assert.equal(p.f.calls.filter(call => call.path === '/me/player/next').length, 2); + assert.equal(p.player.snapshot().title, 'Song c'); + assert.deepEqual(plain(p.player.getQueue().map(item => item.title)), ['Song c']); +}); + +test('volume goes to a speaker through Spotify and to this phone through the system volume', async () => { + const p = playerFixture({ routes: { + 'GET /me/player': { body: playbackBody({ device: { id: 'spk', name: 'Kitchen', type: 'Speaker', is_active: true, volume_percent: 30, supports_volume: true } }) }, + 'PUT /me/player/volume': { status: 204 }, + } }); + p.player.onStateChange(() => {}); + p.timers.advance(0); await p.settle(); + assert.equal(p.player.snapshot().appName, 'Spotify · Kitchen'); + assert.equal(p.player.getMediaVolumePercent(), 30); + p.player.setMediaVolumePercent(32); p.player.setMediaVolumePercent(34); + assert.equal(p.player.getMediaVolumePercent(), 34); + await p.settle(); + const volumes = p.f.calls.filter(call => call.path === '/me/player/volume'); + // One request in flight; the latest value follows it. + assert.deepEqual(volumes.map(call => [call.query.volume_percent, call.query.device_id]), [['32', 'spk'], ['34', 'spk']]); + + const phone = playerFixture(); + phone.player.onStateChange(() => {}); + phone.timers.advance(0); await phone.settle(); + assert.equal(phone.player.getMediaVolumePercent(), 40); + phone.player.setMediaVolumePercent(55); + assert.deepEqual(phone.system.set, [55]); + assert.equal(phone.f.calls.some(call => call.path === '/me/player/volume'), false); +}); + +test('shuffle, repeat and like run their Web API calls and relabel at once', async () => { + const p = playerFixture({ routes: { + 'PUT /me/player/shuffle': { status: 204 }, 'PUT /me/player/repeat': { status: 204 }, 'PUT /me/library': { status: 200 }, + } }); + p.player.onStateChange(() => {}); + p.timers.advance(0); await p.settle(); + await p.player.sendCustomAction('shuffle'); + await p.player.sendCustomAction('repeat'); + await p.player.sendCustomAction('like'); + assert.deepEqual(plain(p.player.snapshot().customActions.map(action => action.name)), ['Shuffle: on', 'Repeat: all', 'Unlike song']); + const calls = p.f.calls.filter(call => call.method === 'PUT'); + assert.deepEqual(calls.map(call => [call.path, call.query.state ?? call.query.uris]), + [['/me/player/shuffle', 'true'], ['/me/player/repeat', 'context'], ['/me/library', 'spotify:track:a']]); +}); + +test('signed out, the player reports no access and never calls the API', async () => { + const p = playerFixture({ settings: fakeSettings({ 'music.spotify.clientId': CLIENT_ID }) }); + p.player.onStateChange(() => {}); + p.timers.advance(0); await p.settle(); + assert.equal(p.player.snapshot().accessEnabled, false); + assert.equal(p.f.calls.length, 0); +}); + +test('the library tree lists owned playlists as browsable and plays tracks within their context', async () => { + const s = loadSpotify({ settings: signedIn() }); + const f = fakeFetch({ + 'GET /me/playlists': { body: { items: [ + { id: 'mine', name: 'Mine', owner: { id: 'jim' } }, + { id: 'theirs', name: 'Theirs', owner: { id: 'spotify', display_name: 'Spotify' } }, + { id: 'collab', name: 'Collab', collaborative: true, owner: { id: 'friend' } }], next: null } }, + 'GET /me/player/recently-played': { body: { items: [ + { track: track('a'), context: { uri: 'spotify:album:al' } }, { track: track('a'), context: null }, { track: track('b'), context: null }] } }, + }); + const plays = []; + const library = new s.library.SpotifyLibrary({ client: new s.api.SpotifyClient(f.fetch, tokens), + player: { startPlayback: async request => { plays.push(request); return true; } }, + isSignedIn: () => true, userId: () => 'jim' }); + const [app] = library.listBrowsableApps(); + assert.equal(await library.connect(app), 'root'); + assert.deepEqual(plain((await library.browse('root')).map(row => row.title)), ['Recently played', 'Playlists', 'Liked Songs', 'Albums', 'Artists']); + const playlists = await library.browse('playlists'); + assert.deepEqual(plain(playlists.map(row => [row.title, row.browsable, row.subtitle])), + [['Mine', true, ''], ['Theirs', false, 'Spotify'], ['Collab', true, '']]); + const recent = await library.browse('recent'); + assert.deepEqual(plain(recent.map(row => row.mediaId)), ['track:spotify:track:a|spotify:album:al', 'track:spotify:track:b']); + + library.playFromMediaId(recent[0].mediaId); + library.playFromMediaId(recent[1].mediaId); + library.playFromMediaId('track:spotify:track:z|liked'); + library.playFromMediaId('liked'); + library.playFromMediaId('playlist:mine'); + assert.deepEqual(plain(plays), [ + { contextUri: 'spotify:album:al', offsetUri: 'spotify:track:a' }, + { uris: ['spotify:track:b'] }, + { contextUri: 'spotify:user:jim:collection', offsetUri: 'spotify:track:z' }, + { contextUri: 'spotify:user:jim:collection' }, + { contextUri: 'spotify:playlist:mine' }, + ]); + library.disconnect(); + await assert.rejects(library.browse('root'), /Not connected/); +}); + +test('search-and-play starts the best match and explains failures', async () => { + const p = playerFixture({ routes: { + 'GET /search': call => ({ body: call.query.type === 'album' + ? { albums: { items: [{ id: 'al', name: 'Discovery', uri: 'spotify:album:al', artists: [{ name: 'Daft Punk' }] }] } } + : { tracks: { items: [] } } }), + 'PUT /me/player/play': { status: 204 }, + } }); + assert.equal(await p.service.playSearch('discovery', 'album'), 'the album Discovery by Daft Punk'); + const play = p.f.calls.find(call => call.path === '/me/player/play'); + assert.deepEqual(JSON.parse(play.body), { context_uri: 'spotify:album:al' }); + await assert.rejects(p.service.playSearch('nothing'), /found no track/); +}); + +test('the service resets the player when the signed-in account changes, not when tokens refresh', async () => { + const p = playerFixture(); + const reset = []; + p.service.player.accountChanged = () => reset.push('reset'); + const changes = []; + p.service.onChange(() => changes.push('change')); + const account = JSON.parse(p.settings.values.get('music.spotify.account')); + p.settings.setStringSetting('music.spotify.account', JSON.stringify({ ...account, accessToken: 'access-9' })); + assert.deepEqual(reset, []); + p.service.signOut(); + assert.deepEqual(reset, ['reset']); + assert.equal(p.service.account(), null); + assert.ok(changes.length >= 2); +}); + +function composite(appleState, spotifyState, spotifyLastPlayingAtMs = 0) { + const types = load('app/native/media-types.ts'); + const calls = []; + const fakeBridge = (name, state) => ({ + accessLabel: `${name} label`, accessAction: `${name} action`, skipRebuildsQueue: name === 'spotify', state, lastPlayingAtMs: 0, + onStateChange(listener) { listener(this.state); return () => {}; }, + snapshot() { return this.state; }, start: async () => calls.push(`${name}:start`), stop: async () => {}, + playPause: async () => calls.push(`${name}:playPause`), skipNext: async () => calls.push(`${name}:next`), + getAlbumArt: () => name, getQueue: () => [], refresh: async () => calls.push(`${name}:refresh`), + hasAccess: () => name === 'apple', requestAccess: async () => calls.push(`${name}:access`), + }); + const apple = fakeBridge('apple', appleState); + const spotify = fakeBridge('spotify', spotifyState); + spotify.lastPlayingAtMs = spotifyLastPlayingAtMs; + const api = load('app/native/media-controller.ios.ts', { + '@nativescript/core': { Dialogs: {}, Utils: {} }, './image-files': {}, './kotlin-data': {}, './media-types': types, + './spotify.ios': { spotifyService: () => null }, + }); + const bridge = new api.IosMediaControllerBridge(apple, () => (spotifyState ? spotify : null), () => 50); + return { api, bridge, calls, apple, spotify, types }; +} + +test('the iOS bridge follows whichever player is playing and routes commands to it', async () => { + const types = load('app/native/media-types.ts'); + const base = { ...types.DEFAULT_MEDIA_STATE, accessEnabled: true }; + const playing = { ...base, available: true, playbackState: 'playing', title: 'Playing' }; + const paused = { ...base, available: true, playbackState: 'paused', title: 'Paused' }; + const { chooseMediaSource } = composite(base, null).api; + assert.equal(chooseMediaSource('apple-music', base, null, 0, 0), 'apple-music'); + assert.equal(chooseMediaSource('apple-music', { ...base, accessEnabled: false }, base, 0, 0), 'spotify'); + assert.equal(chooseMediaSource('apple-music', paused, playing, 0, 0), 'spotify'); + assert.equal(chooseMediaSource('spotify', playing, paused, 0, 0), 'apple-music'); + assert.equal(chooseMediaSource('spotify', playing, playing, 0, 0), 'spotify'); // no flapping + assert.equal(chooseMediaSource('spotify', paused, paused, 20, 10), 'apple-music'); // played last + assert.equal(chooseMediaSource('apple-music', paused, paused, 0, 0), 'spotify'); + assert.equal(chooseMediaSource('apple-music', paused, base, 0, 0), 'apple-music'); + + const c = composite(paused, { ...playing, notice: '' }); + assert.equal(c.bridge.snapshot().title, 'Playing'); + assert.equal(c.bridge.skipRebuildsQueue, true); + await c.bridge.playPause(); await c.bridge.refresh(); + assert.deepEqual(c.calls, ['spotify:playPause', 'spotify:refresh']); + assert.equal(c.bridge.getAlbumArt(10), 'spotify'); + const firstArt = c.bridge.snapshot().artVersion; + // Switching players changes the art version even when the sources' versions match. + c.spotify.state = { ...paused, notice: 'Spotify needs Premium' }; + c.apple.state = { ...playing, title: 'Apple' }; + const state = c.bridge.snapshot(); + assert.equal(state.title, 'Apple'); assert.notEqual(state.artVersion, firstArt); + // A Spotify failure still shows while Apple Music is the player shown. + assert.equal(state.notice, 'Spotify needs Premium'); + assert.equal(c.bridge.hasAccess(), true); // Apple Music's permission + await c.bridge.requestAccess(); + assert.equal(c.calls.at(-1), 'apple:access'); + + const idle = composite(base, base); + assert.equal(idle.bridge.snapshot().status, 'Nothing is playing.'); +}); + +test('the assistant offers Spotify search-and-play once signed in and refreshes before reading media state', async () => { + let account = null; const searches = []; const refreshes = []; + const spotify = { + account: () => account, + playSearch: async (query, kind) => { searches.push([query, kind]); if (query === 'fail') throw new Error('No device'); return 'Song by Band'; }, + }; + const registered = new Map(); const systemTools = new Map(); + const { registerSystemTools } = load('app/assistant/system-tools.ts', { + '../native/calendar': {}, '../native/calendar-permissions': {}, + '../native/media-controller': { mediaControllerBridge: { start: async () => {}, refresh: async () => refreshes.push('refresh'), + snapshot: () => ({ accessEnabled: true, playbackState: 'playing', title: 'Song', artist: 'Band', appName: 'Spotify' }) } }, + '../native/spotify': { spotifyService: () => spotify }, + '../native/notification-icons': {}, '../ui/shell/shell': {}, './tool-registry': {}, + }, { global: { isIOS: true } }); + registerSystemTools({ + registerSystemTool: (definition, handler) => systemTools.set(definition.name, handler), + register: registration => registered.set(registration.spec.name, registration), + }); + assert.match((await systemTools.get('media.now_playing')()).content, /Playing: Song by Band in Spotify/); + assert.deepEqual(refreshes, ['refresh']); + + const tool = registered.get('media.spotify_play'); + assert.equal(tool.spec.availability, 'always'); + assert.equal(tool.isAvailable(), false); + account = { userId: 'jim' }; + assert.equal(tool.isAvailable(), true); + assert.equal((await tool.handler({ query: ' discovery ', kind: 'album' })).content, 'Playing Song by Band on Spotify.'); + await tool.handler({ query: 'x', kind: 'podcast' }); + assert.deepEqual(plain(searches), [['discovery', 'album'], ['x', 'track']]); + assert.equal((await tool.handler({ query: 'fail' })).error, 'No device'); + assert.equal((await tool.handler({ query: ' ' })).ok, false); +}); diff --git a/tests/native/ios-loopback-redirect-test.m b/tests/native/ios-loopback-redirect-test.m new file mode 100644 index 00000000..04221716 --- /dev/null +++ b/tests/native/ios-loopback-redirect-test.m @@ -0,0 +1,69 @@ +#import +#import "FaceclawLoopbackRedirect.h" + +/** GET a URL, pumping the main run loop (where the receiver delivers redirects) until it answers. */ +static NSInteger get(NSString *url, NSString **body) { + __block NSInteger status = -1; + __block NSString *text = nil; + __block BOOL done = NO; + [[NSURLSession.sharedSession dataTaskWithURL:[NSURL URLWithString:url] + completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) { + status = error ? -1 : ((NSHTTPURLResponse *)response).statusCode; + text = data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : nil; + done = YES; + }] resume]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10]; + while (!done && deadline.timeIntervalSinceNow > 0) { + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]]; + } + if (body) *body = text; + return status; +} + +static void pumpBriefly(void) { + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.2]]; +} + +int main(int argc, const char **argv) { + @autoreleasepool { + NSInteger port = argc > 1 ? atoi(argv[1]) : 43899; + NSString *base = [NSString stringWithFormat:@"http://127.0.0.1:%ld", (long)port]; + FaceclawLoopbackRedirect *receiver = [FaceclawLoopbackRedirect new]; + NSMutableArray *redirects = [NSMutableArray array]; + receiver.redirectHandler = ^(NSString *url) { [redirects addObject:url]; }; + NSString *error = [receiver startOnPort:port path:@"/spotify-callback"]; + if (error) { NSLog(@"FAIL start: %@", error); return 1; } + + // A second listener on the same port reports the conflict instead of failing silently. + FaceclawLoopbackRedirect *second = [FaceclawLoopbackRedirect new]; + NSString *conflict = [second startOnPort:port path:@"/spotify-callback"]; + if (!conflict || [conflict rangeOfString:@"port"].location == NSNotFound) { NSLog(@"FAIL conflict: %@", conflict); return 1; } + + // Other paths (a browser's favicon request) are refused and not reported. + NSString *body = nil; + if (get([base stringByAppendingString:@"/favicon.ico"], &body) != 404) { NSLog(@"FAIL favicon"); return 1; } + pumpBriefly(); + if (redirects.count) { NSLog(@"FAIL favicon reported %@", redirects); return 1; } + + // The redirect itself: a page for the browser, the full URL (query intact) for the app. + NSString *redirect = [base stringByAppendingString:@"/spotify-callback?code=AQB%2Fx-y_z&state=s1"]; + if (get(redirect, &body) != 200 || [body rangeOfString:@"return to Faceclaw"].location == NSNotFound) { + NSLog(@"FAIL redirect page: %@", body); return 1; + } + pumpBriefly(); + if (redirects.count != 1 || ![redirects[0] isEqualToString:redirect]) { NSLog(@"FAIL redirect url %@", redirects); return 1; } + + // A query-less or prefixed path does not count as the redirect. + if (get([base stringByAppendingString:@"/spotify-callback-extra?code=x"], NULL) != 404) { NSLog(@"FAIL prefix"); return 1; } + + // Once stopped, the port is free again. + [receiver stop]; + pumpBriefly(); + if (get(redirect, NULL) != -1) { NSLog(@"FAIL still listening after stop"); return 1; } + NSString *again = [second startOnPort:port path:@"/spotify-callback"]; + if (again) { NSLog(@"FAIL restart: %@", again); return 1; } + [second stop]; + NSLog(@"PASS loopback redirect"); + return 0; + } +} From 0b7d93a3f16c1147159f447cc9f2ce8b30058105 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 13:12:25 -0700 Subject: [PATCH 02/34] Translate app: sound level meter; translate unfinished segments --- app/apps/microphones/mic-session.ts | 20 +-- app/apps/transcribe/transcribe-app.ts | 2 +- app/apps/transcribe/transcribe.ts | 12 +- app/apps/translate/translate-app.ts | 18 ++- app/apps/translate/translate-llm.ts | 4 +- app/apps/translate/translate-settings.ts | 52 +++++++ app/apps/translate/translate.ts | 55 ++++--- app/apps/translate/translation-model.ts | 175 +++++++++++++++++------ app/g2/dashboard-controller.ts | 23 +-- app/g2/ios-preview-controller.ts | 21 +-- app/native/cloud-stt-provider.ts | 21 +-- app/native/cloud-stt.ts | 2 + app/native/elevenlabs-stt.ts | 5 +- app/native/openai-stt.ts | 6 +- app/native/soniox-stt.ts | 2 + app/native/voice-control.ios.ts | 5 + app/native/voice-control.ts | 19 ++- app/prompts.ts | 9 +- app/ui/layers.ts | 12 +- app/ui/listening-header.ts | 92 ++++++++++++ app/util/audio-level.ts | 24 ++++ tests/cloud-stt.test.cjs | 28 +++- tests/ios-app-host.test.cjs | 15 +- tests/listening-header.test.cjs | 100 +++++++++++++ tests/translate.test.cjs | 132 ++++++++++++++--- 25 files changed, 679 insertions(+), 175 deletions(-) create mode 100644 app/ui/listening-header.ts create mode 100644 app/util/audio-level.ts create mode 100644 tests/listening-header.test.cjs diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index 5139f09b..88aedc2d 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment"; import { generateSessionInsights } from "./speaker-insights"; import { speakerRegistry, type SpeakerMatch } from "./speakers"; import { postAmbientCard } from "../../ui/shell/ambient-cards"; +import { pcmLevel, rmsToLevel } from "../../util/audio-level"; import { formatRelativeTime } from "../../util/date-util"; import { deviceLanguage, identifyLanguage, translateText } from "./translate"; @@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig { return { ...config, channelMask: 0b11 }; } -function pcmLevel(pcm: Uint8Array): number { - let sumSquares = 0; - const count = pcm.length / 2; - if (count === 0) return 0; - for (let i = 0; i < count; i++) { - let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); - if (value >= 0x8000) value -= 0x10000; - sumSquares += value * value; - } - return rmsToLevel(Math.sqrt(sumSquares / count) / 32768); -} - function floatLevel(samples: Float32Array): number { let sumSquares = 0; for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!; return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length))); } -/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */ -function rmsToLevel(rms: number): number { - if (rms <= 0) return 0; - const db = 20 * Math.log10(rms); - return Math.max(0, Math.min(1, (db + 60) / 60)); -} - function floatToPcm16(samples: Float32Array): Uint8Array { const out = new Uint8Array(samples.length * 2); for (let i = 0; i < samples.length; i++) { diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index 71c1b4c6..900af357 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc }); // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. - layer.start(app.requestRender); + layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID)); options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); return app; diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 251b1807..2fd35a6c 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -1,6 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; +import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer { private detail = ""; private readonly transcript = new TranscriptModel(); private readonly scroll = new TranscriptScroll(); + private readonly meter = new MicLevelMeter(); private saveNotice = ""; private unsubscribeTranscript: (() => void) | null = null; private unsubscribePause: (() => void) | null = null; @@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer { /** `stamp` names the saved transcript, matching the session's audio recording. */ constructor(private readonly stamp: string = transcriptTimestamp()) {} - start(requestRender: () => void): void { + /** `isVisible` says whether the window is on screen (the level meter repaints only then). */ + start(requestRender: () => void, isVisible: () => boolean): void { + this.meter.start(requestRender, isVisible); this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { this.transcript.accept(event); requestRender(); @@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer { const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64); - image.drawText(font, 24, 20, "Transcribe", 200); - image.drawText(font, 24, 40, this.saveNotice || this.status, 110); - + const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null); const step = lineStep(font) + 2; - const bodyTop = 62; const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); const firstLine = this.scroll.layout(wrapped.length, bodyLines); for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) { @@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer { } onRemoved(): void { + this.meter.stop(); this.unsubscribePause?.(); this.unsubscribePause = null; this.unsubscribeTranscript?.(); diff --git a/app/apps/translate/translate-app.ts b/app/apps/translate/translate-app.ts index 7cbd57e3..1cb11a4a 100644 --- a/app/apps/translate/translate-app.ts +++ b/app/apps/translate/translate-app.ts @@ -1,6 +1,12 @@ import { TranslateLayer } from "./translate"; import { resolveTranslationModel } from "./translate-llm"; -import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings"; +import { + resolveTranslateLanguage, + resolveTranslateProvider, + translateLanguageSetting, + translateProviderSetting, + translateShowOriginalSetting, +} from "./translate-settings"; import { MIC_ICON } from "../transcribe/transcribe-app"; import { transcriptTimestamp } from "../transcribe/transcribe"; import { type ContinuousCaptureOptions } from "../../ui/layers"; @@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & { * provider (Translate settings), which a shared capture switches to. */ export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow { - // Checks the setup and (re)starts capture with the chosen provider: at - // launch, when the provider setting changes, and on a tap after a problem. + // Checks the setup and (re)starts capture with the chosen provider and + // language: at launch, when either setting changes, and on a tap after a + // problem. const startCapture = () => { const provider = resolveTranslateProvider(); const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null; @@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces shell.setTrayIcon(TRAY_ICON_ID, null); return; } - options.startContinuousVoiceCapture({ multilingualProvider: provider }); + options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); }; const layer = new TranslateLayer(transcriptTimestamp(), startCapture); @@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces onSelect: (ctx) => { ctx.stack.pop(); openSettingsSubMenu(ctx, "Translate settings", [ + enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }), enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }), toggleSettingMenuItem(translateShowOriginalSetting), ]); @@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces }); // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. - layer.start(app.requestRender); + layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID)); startCapture(); return app; } diff --git a/app/apps/translate/translate-llm.ts b/app/apps/translate/translate-llm.ts index 26021899..70883340 100644 --- a/app/apps/translate/translate-llm.ts +++ b/app/apps/translate/translate-llm.ts @@ -29,6 +29,8 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null { export type TranslationRequest = { speech: string; context: TranslationTurn[]; + /** English name of the spoken language when the user set one, else "". */ + language: string; onProgress: (textSoFar: string) => void; onDone: (translation: string) => void; onError: (message: string) => void; @@ -41,7 +43,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl system: TRANSLATE_SYSTEM_PROMPT, effort: model.effort, maxTokens: 1024, - messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }], + messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }], onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)), onDone: (result) => { if (result.stopReason === "refusal") request.onError("The model declined to translate this."); diff --git a/app/apps/translate/translate-settings.ts b/app/apps/translate/translate-settings.ts index c75d1ef3..802419e1 100644 --- a/app/apps/translate/translate-settings.ts +++ b/app/apps/translate/translate-settings.ts @@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum "Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.", }); +/** + * Spoken languages Translate can be told to expect, by ISO 639-1 code (the + * form every multilingual provider accepts), in picker order. + */ +const SPOKEN_LANGUAGES = { + ar: "Arabic", + zh: "Chinese", + nl: "Dutch", + fr: "French", + de: "German", + el: "Greek", + he: "Hebrew", + hi: "Hindi", + id: "Indonesian", + it: "Italian", + ja: "Japanese", + ko: "Korean", + pl: "Polish", + pt: "Portuguese", + ru: "Russian", + es: "Spanish", + sv: "Swedish", + th: "Thai", + tr: "Turkish", + uk: "Ukrainian", + vi: "Vietnamese", +} as const; + +export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES; + +export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({ + id: "translate-language", + label: "Spoken language", + storageKey: "translate.language", + defaultValue: "auto", + values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])], + formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]), + description: + "The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.", +}); + +/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */ +export function resolveTranslateLanguage(): SpokenLanguage | "" { + const value = translateLanguageSetting.get(); + return value === "auto" ? "" : value; +} + +/** The English name of a spoken language, for the translation prompt. */ +export function spokenLanguageName(language: SpokenLanguage): string { + return SPOKEN_LANGUAGES[language]; +} + export const translateShowOriginalSetting = new ConfigSettingBoolean({ id: "translate-show-original", label: "Show original", diff --git a/app/apps/translate/translate.ts b/app/apps/translate/translate.ts index 252da0d4..e747be2d 100644 --- a/app/apps/translate/translate.ts +++ b/app/apps/translate/translate.ts @@ -1,6 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; +import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol"; import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model"; import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model"; import { resolveTranslationModel, streamTranslation } from "./translate-llm"; -import { translateShowOriginalSetting } from "./translate-settings"; +import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings"; const NO_MODEL_NOTICE = "No language model is set up."; /** * Live translation view. Continuous mic capture is owned by the app wrapper - * (createTranslateAppWindow); this layer feeds finalized speech to the - * assistant's model one stretch at a time, shows the English as it streams - * in, and saves the session to Downloads from the app menu. + * (createTranslateAppWindow); this layer feeds speech to the assistant's + * model one block at a time (provisionally while a block is still being + * spoken), shows the English as it streams in, and saves the session to + * Downloads from the app menu. */ export class TranslateLayer implements Layer { private status = "Starting microphone..."; @@ -30,7 +32,8 @@ export class TranslateLayer implements Layer { private problem: string | null = null; private readonly model = new TranslationModel(); private readonly scroll = new TranscriptScroll(); - private activeJob: TranslationJob | null = null; + private readonly meter = new MicLevelMeter(); + // The request in flight (the model tracks which job it is). private activeHandle: LlmStreamHandle | null = null; private requestRender: () => void = () => {}; private removed = false; @@ -40,10 +43,16 @@ export class TranslateLayer implements Layer { /** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */ constructor(private readonly stamp: string, private readonly retry: () => void) {} - start(requestRender: () => void): void { + /** `isVisible` says whether the window is on screen (the level meter repaints only then). */ + start(requestRender: () => void, isVisible: () => boolean): void { this.requestRender = requestRender; + this.meter.start(requestRender, isVisible); this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { - this.model.accept(event); + if (this.model.accept(event)) { + // The request in flight translates text the transcript has replaced. + this.activeHandle?.cancel(); + this.activeHandle = null; + } this.pump(); requestRender(); }); @@ -68,9 +77,9 @@ export class TranslateLayer implements Layer { const measure = (line: string) => font.measureText(line); const maxWidth = width - 64; - image.drawText(font, 24, 20, "Translate", 200); const status = this.problem ? "Setup needed" : this.translationError || this.status; - image.drawText(font, 24, 40, this.saveNotice || status, 110); + const meter = this.listening && !this.problem ? this.meter : null; + const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter); let lines: TranslationLine[]; if (this.problem) { @@ -84,7 +93,6 @@ export class TranslateLayer implements Layer { } const step = lineStep(font) + 2; - const bodyTop = 62; const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); const firstLine = this.scroll.layout(lines.length, bodyLines); for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) { @@ -119,7 +127,7 @@ export class TranslateLayer implements Layer { onRemoved(): void { this.removed = true; - this.activeJob = null; + this.meter.stop(); this.activeHandle?.cancel(); this.activeHandle = null; this.unsubscribeTranscript?.(); @@ -139,7 +147,10 @@ export class TranslateLayer implements Layer { this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } - /** Start translating the next waiting stretch of speech, unless one is in flight. */ + /** + * Start the next translation request (a final block, else the live block's + * provisional one), unless one is in flight. + */ private pump(): void { if (this.removed) return; const job = this.model.nextJob(); @@ -147,36 +158,34 @@ export class TranslateLayer implements Layer { const llm = resolveTranslationModel(); if (!llm) { // The model went away mid-session (a key was cleared): keep the original. - this.model.fail(job.segment); + this.model.fail(job); this.translationError = NO_MODEL_NOTICE; this.pump(); return; } - this.activeJob = job; + const language = resolveTranslateLanguage(); const handle = streamTranslation(llm, { - speech: job.segment.source, + speech: job.source, context: job.context, + language: language ? spokenLanguageName(language) : "", onProgress: (textSoFar) => { - if (this.activeJob !== job) return; - this.model.progress(job.segment, textSoFar); - this.requestRender(); + if (this.model.progress(job, textSoFar)) this.requestRender(); }, onDone: (translation) => this.settle(job, () => { - this.model.complete(job.segment, translation); + this.model.complete(job, translation); this.translationError = ""; }), onError: (message) => this.settle(job, () => { - this.model.fail(job.segment); + this.model.fail(job); this.translationError = `Translation failed: ${message}`; }), }); // A provider that fails synchronously has already settled (and moved on). - if (this.activeJob === job) this.activeHandle = handle; + if (this.model.isActive(job)) this.activeHandle = handle; } private settle(job: TranslationJob, apply: () => void): void { - if (this.activeJob !== job) return; - this.activeJob = null; + if (!this.model.isActive(job)) return; this.activeHandle = null; apply(); this.pump(); diff --git a/app/apps/translate/translation-model.ts b/app/apps/translate/translation-model.ts index ea0e9204..2a1fc9da 100644 --- a/app/apps/translate/translation-model.ts +++ b/app/apps/translate/translation-model.ts @@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model"; type TranscriptEvent = { text: string; isFinal: boolean }; -export type TranslationState = "pending" | "translating" | "done" | "failed"; - -/** One stretch of finalized speech and its English translation. */ +/** + * One block of speech (up to a pause) and its English translation. The last + * block may still be live, its transcript revising; it is translated + * provisionally as it grows, then again once final if its text changed. + */ export type TranslationSegment = { source: string; - /** The translation so far (streamed while translating). */ + /** Committed by the transcriber; a live block's source still revises. */ + final: boolean; + /** The latest finished translation, of `translatedSource`. */ translation: string; - state: TranslationState; + /** What `translation` translates: null until one finishes, behind `source` while provisional. */ + translatedSource: string | null; + /** The text streamed so far by the request in flight for this block, else null. */ + streaming: string | null; + /** A source whose translation failed; not retried until the source changes. */ + failedSource: string | null; }; export type TranslationTurn = { source: string; translation: string }; -/** A segment handed to the model, with the finished turns before it. */ +/** A request: a block's source as sent, with the finished turns before it. */ export type TranslationJob = { segment: TranslationSegment; + /** The source sent (a live block's moves on while the request is in flight). */ + source: string; /** Earlier translated turns, oldest first. */ context: TranslationTurn[]; }; @@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number }; // over between requests; bounded so a long session's requests stay small. const CONTEXT_TURNS = 4; const CONTEXT_CHARS = 800; +// A live block is translated again once its text has changed length by this +// much since the last request, so a trickle of revisions (punctuation, one +// corrected word) doesn't spend a request each. A final is always redone. +const PROVISIONAL_MIN_CHANGE = 3; const TRANSLATION_GRAY = 230; +const PROVISIONAL_GRAY = 170; const ORIGINAL_GRAY = 120; const LIVE_GRAY = 90; +function newSegment(source: string, final: boolean): TranslationSegment { + return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null }; +} + +/** Whether a block's current source still needs a translation request. */ +function wantsTranslation(segment: TranslationSegment): boolean { + const { source, translatedSource } = segment; + if (!source || source === translatedSource || source === segment.failedSource) return false; + return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE; +} + /** - * The Translate app's conversation: finalized speech segments with their - * translations, plus the live partial of the utterance in progress. + * The Translate app's conversation: blocks of speech with their translations. * Translation is serial (one request at a time, in order, so each can carry - * the turns before it as context); finals that arrive while a request is in - * flight merge into one pending segment, so a slow model batches the backlog - * instead of falling further behind one request per pause. + * the turns before it as context): final blocks first, then the live one. + * Finals that arrive while a request is in flight merge into one waiting + * block, so a slow model batches the backlog instead of falling further + * behind one request per pause. */ export class TranslationModel { readonly segments: TranslationSegment[] = []; - /** The utterance in progress (replace semantics), not yet final. */ - live = ""; + private active: TranslationJob | null = null; - accept(event: TranscriptEvent): void { + /** + * Take a transcript event (replace semantics within the live block). + * Returns true when it made the request in flight moot (its block was + * finalized as other text, or dropped); that request is abandoned here, and + * the caller should cancel it. + */ + accept(event: TranscriptEvent): boolean { const text = event.text.trim(); - if (!event.isFinal) { - this.live = text; - return; - } - const finalText = text || this.live; - this.live = ""; - if (!finalText) return; const last = this.segments[this.segments.length - 1]; - if (last?.state === "pending") last.source = `${last.source} ${finalText}`; - else this.segments.push({ source: finalText, translation: "", state: "pending" }); + const live = last && !last.final ? last : null; + if (!event.isFinal) { + if (live) live.source = text; + else if (text) this.segments.push(newSegment(text, false)); + return false; + } + const finalText = text || live?.source || ""; + if (!live) { + if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true)); + return false; + } + const moot = this.active?.segment === live && this.active.source !== finalText; + if (moot) this.abandonActive(); + if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) { + this.segments.pop(); + return moot; + } + live.source = finalText; + live.final = true; + return moot; } - /** Take the next segment to translate, or null while one is in flight or none waits. */ + /** Take the next request to make, or null while one is in flight or nothing waits. */ nextJob(): TranslationJob | null { - if (this.segments.some((segment) => segment.state === "translating")) return null; - const segment = this.segments.find((candidate) => candidate.state === "pending"); + if (this.active) return null; + const segment = this.segments.find((candidate) => wantsTranslation(candidate)); if (!segment) return null; - segment.state = "translating"; - return { segment, context: this.contextBefore(segment) }; + segment.streaming = ""; + this.active = { segment, source: segment.source, context: this.contextBefore(segment) }; + return this.active; } - progress(segment: TranslationSegment, textSoFar: string): void { - if (segment.state === "translating") segment.translation = textSoFar; + isActive(job: TranslationJob): boolean { + return this.active === job; } - complete(segment: TranslationSegment, translation: string): void { - segment.translation = translation; - segment.state = "done"; + /** Streamed text for the request in flight; false when `job` isn't it any more. */ + progress(job: TranslationJob, textSoFar: string): boolean { + if (this.active !== job) return false; + job.segment.streaming = textSoFar; + return true; } - fail(segment: TranslationSegment): void { - segment.translation = ""; - segment.state = "failed"; + complete(job: TranslationJob, translation: string): void { + if (this.active !== job) return; + this.active = null; + job.segment.streaming = null; + job.segment.translation = translation; + job.segment.translatedSource = job.source; + } + + fail(job: TranslationJob): void { + if (this.active !== job) return; + this.active = null; + job.segment.streaming = null; + job.segment.failedSource = job.source; + } + + private abandonActive(): void { + if (this.active) this.active.segment.streaming = null; + this.active = null; + } + + /** + * Append a final to the block before `before` (default: the end) when that + * block is final but still waiting for its first request. Returns whether + * it did. + */ + private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean { + const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1; + const waiting = this.segments[index]; + if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false; + if (this.active?.segment === waiting) return false; + waiting.source = `${waiting.source} ${finalText}`; + return true; } private contextBefore(segment: TranslationSegment): TranslationTurn[] { @@ -87,7 +162,7 @@ export class TranslationModel { let chars = 0; for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) { const earlier = this.segments[index]!; - if (earlier.state !== "done" || !earlier.translation) continue; + if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue; chars += earlier.source.length + earlier.translation.length; if (turns.length && chars > CONTEXT_CHARS) break; turns.unshift({ source: earlier.source, translation: earlier.translation }); @@ -96,9 +171,11 @@ export class TranslationModel { } /** - * Display lines: each segment's translation, bright; its original, dim, - * above it when `showOriginal` is on and in its place until a translation - * arrives (or when it failed); then the live partial, dimmer. + * Display lines: each block's translation, bright once settled (final, and + * of its final text) and dimmer while provisional; its original, dim, above + * it when `showOriginal` is on and in its place until a translation + * arrives. A redo streams in over the translation it replaces only once it + * is as long, so the text doesn't shrink and regrow. */ lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] { const lines: TranslationLine[] = []; @@ -106,21 +183,25 @@ export class TranslationModel { for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray }); }; for (const segment of this.segments) { - const translation = segment.translation.trim(); - // A finished, empty translation is noise the model chose to drop. - if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY); - if (translation) push(translation, TRANSLATION_GRAY); + if (!segment.source) continue; + const settled = segment.final && segment.translatedSource === segment.source; + const finished = segment.translation.trim(); + const streaming = segment.streaming?.trim() ?? ""; + const translation = streaming.length >= finished.length ? streaming : finished; + // A settled, empty translation is noise the model chose to drop. + if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY); + if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY); } - if (this.live) push(this.live, LIVE_GRAY); return lines; } - /** Plain-text transcript for saving: each original, then its translation. */ + /** Plain-text transcript for saving: each final original, then its translation. */ savedText(): string { return this.segments + .filter((segment) => segment.final) .map((segment) => { const translation = segment.translation.trim(); - if (segment.state === "failed") return `${segment.source}\n→ (not translated)`; + if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`; if (!translation || translation === segment.source.trim()) return segment.source; return `${segment.source}\n→ ${translation}`; }) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index db83e5b2..a04781f6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -74,9 +74,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; -import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; +import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -2116,10 +2116,10 @@ class DashboardController { voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator)); } - /** The provider a continuous-capture window needs (Translate's multilingual one), if any. */ - private multilingualProvider(): VoiceProvider | undefined { + /** The provider and language a continuous-capture window needs (Translate's), if any. */ + private transcriptionChoice(): TranscriptionChoice | undefined { for (const options of this.continuousClients.values()) { - if (options.multilingualProvider) return options.multilingualProvider; + if (options.transcription) return options.transcription; } return undefined; } @@ -2144,20 +2144,21 @@ class DashboardController { * stands in for the G2 mic. */ private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { - // A window that needs a multilingual provider sets it for every capture - // while it is open, so push-to-talk still shares its stream. - const multilingual = this.multilingualProvider(); + // A window that needs its own provider and language sets them for every + // capture while it is open, so push-to-talk still shares its stream. + // Otherwise only Whisper server is told a language (its own setting). + const choice = this.transcriptionChoice(); + const provider = choice?.provider ?? voiceProviderSetting.get(); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, - provider: multilingual ?? voiceProviderSetting.get(), + provider, + language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "", elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), - // Whisper detects the language itself when the speech may be in any. - whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 982f0904..b3fe04a5 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -149,11 +149,12 @@ export class IosPreviewController { private logLines: string[] = [] private logTimer: ReturnType | null = null // Windows that want continuous capture, and what each needs of it (a - // recording, a multilingual provider). They share one capture, which runs + // recording, a provider and language). They share one capture, which runs // while any of them wants it. private readonly continuousClients = new Map() - // transcriptionKey of the running continuous capture's provider; one that no - // longer matches what its windows need is drained and restarted. + // transcriptionKey of the running continuous capture (provider and + // language); one that no longer matches what its windows need is drained + // and restarted. private continuousTranscription = '' // Continuous capture should (re)start once audio is available: a window // asked, the glasses connected or disconnected, or the glasses lock or the @@ -1058,15 +1059,17 @@ export class IosPreviewController { } /** * The transcription provider setting and the keys the cloud ones need. A - * window that needs a multilingual provider (Translate) sets it for every - * capture while it is open, with Whisper left to detect the language. + * window that needs its own provider and language (Translate) sets them for + * every capture while it is open; otherwise only Whisper server is told a + * language (its own setting). */ private voiceProvider(): VoiceProviderSelection { - const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider - return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription + const provider = choice?.provider ?? voiceProviderSetting.get() + return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '', + elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), - whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } + whisperServerApiKey: whisperServerApiKeySetting.get() } } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 2ab6ea08..566cb1d7 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ */ export type VoiceProviderSelection = { provider: VoiceProviderKind; + /** + * Spoken language the provider should expect (ISO 639-1); empty lets it + * detect the language. Cloud providers only. + */ + language?: string; elevenLabsApiKey: string; openAiApiKey: string; sonioxApiKey: string; /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ whisperServerUrl?: string; whisperServerModel?: string; - whisperServerLanguage?: string; whisperServerApiKey?: string; }; @@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial> = { create: (options, selection) => new WhisperServerSttClient(options, { baseUrl: selection.whisperServerUrl ?? "", model: selection.whisperServerModel ?? "", - language: selection.whisperServerLanguage ?? "", + language: selection.language ?? "", }), finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS, }, @@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection } /** - * What a capture transcribes with: the provider, and for Whisper server the - * language it is told to expect. A running capture is shared only by holders - * asking for the same. + * What a capture transcribes with: the provider and the language it is told + * to expect. A running capture is shared only by holders asking for the same. */ -export function transcriptionKey(selection: Pick): string { - return selection.provider === "whisper-server" - ? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}` - : selection.provider; +export function transcriptionKey(selection: Pick): string { + return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ @@ -105,7 +106,7 @@ export function createCloudSttClient( const apiKey = provider.apiKey(selection).trim(); return new ReconnectingSttClient( (clientOptions) => provider.create(clientOptions, selection), - { ...options, apiKey }, + { ...options, apiKey, language: (selection.language ?? "").trim() }, continuous, provider.finalTimeoutMs, ); diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index 2600f802..40d4629f 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = { export type CloudSttOptions = { apiKey: string; + /** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */ + language?: string; onTranscript: (event: CloudSttTranscriptEvent) => void; onStatus: (status: string) => void; onError: (message: string) => void; diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index bc493fb9..8b8ee545 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -33,7 +33,10 @@ export class ElevenLabsSttClient { start(): void { if (this.closed || this.ws) return; - const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; + // Without language_code the model detects the language, which can go + // badly wrong on short or noisy segments (Japanese heard as Russian). + const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : ""; + const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`; const listener = { onOpen: () => { if (this.closed) return; diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 063e0a39..24f109a1 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { onOpen: () => { if (this.closed) return; this.open = true; - this.trySend(JSON.stringify(sessionConfig())); + this.trySend(JSON.stringify(sessionConfig(this.options.language ?? ""))); for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } @@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } -function sessionConfig(): object { +function sessionConfig(language: string): object { return { type: "session.update", session: { @@ -209,7 +209,7 @@ function sessionConfig(): object { audio: { input: { format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, - transcription: { model: MODEL_ID }, + transcription: { model: MODEL_ID, ...(language ? { language } : {}) }, // Push-to-talk owns the utterance boundary; no server VAD. turn_detection: null, }, diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts index aa0d210c..fc24f2e0 100644 --- a/app/native/soniox-stt.ts +++ b/app/native/soniox-stt.ts @@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient { sample_rate: CLOUD_STT_SAMPLE_RATE, num_channels: 1, enable_speaker_diarization: true, + // A hint, not a restriction: other languages are still recognized. + ...(this.options.language ? { language_hints: [this.options.language] } : {}), }), ); if (this.closed) return; diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 38dd996f..f5a66124 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -3,6 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt' import type { ReconnectingSttClient } from './reconnecting-stt' import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' import { WavRecorder } from './wav-recorder.ios' +import { pcmLevel } from '../util/audio-level' declare const FaceclawSpeech: any export type IosMicrophoneSession = { @@ -24,6 +25,7 @@ export class IosVoiceControlBridge { private readonly statuses = new Set<(state: VoiceControlState) => void>() private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>() private readonly ends = new Set<() => void>() + private readonly levels = new Set<(level: number) => void>() private status = 'Voice input ready.' private generation = 0 private session: IosMicrophoneSession | null = null @@ -60,6 +62,8 @@ export class IosVoiceControlBridge { } onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } } onSpeechPause(_listener: () => void): () => void { return () => {} } + /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */ + onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } } private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) } private ensureNative(): any { if (!this.native) { @@ -69,6 +73,7 @@ export class IosVoiceControlBridge { const bytes = new Uint8Array(interop.bufferFromData(pcm)) this.cloud?.acceptPcm(bytes) if (this.continuous) this.record(bytes) + if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) } } } return this.native diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index c80d98ca..038c8ceb 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { SpeechPauseDetector } from "./speech-pause"; import { toUint8Array } from "../util/array-util"; +import { pcmLevel } from "../util/audio-level"; declare const com: any; @@ -56,8 +57,9 @@ export type PushToTalkOptions = { sonioxApiKey: string; whisperServerUrl?: string; whisperServerModel?: string; - whisperServerLanguage?: string; whisperServerApiKey?: string; + /** Spoken language for cloud providers (see VoiceProviderSelection.language). */ + language?: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free @@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge { // stopped for its duration. private readonly rawPcmListeners = new Set(); private readonly frameMetaListeners = new Set(); + private readonly micLevelListeners = new Set<(level: number) => void>(); private rawActive = false; // Who wants the raw tap. It runs while any owner does, so one owner letting // go (an EvenHub app backgrounding) doesn't silence another. @@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge { return () => this.rawPcmListeners.delete(listener); } + /** + * Subscribe to the mic level (meter position in [0,1], see pcmLevel) of + * each decoded audio chunk, about every 50 ms, from any capture: STT in + * every mode, or the raw tap. + */ + onMicLevel(listener: (level: number) => void): () => void { + this.micLevelListeners.add(listener); + return () => this.micLevelListeners.delete(listener); + } + /** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */ onFrameMeta(listener: FrameMetaListener): () => void { this.frameMetaListeners.add(listener); @@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge { if (this.rawPcmListeners.size > 0) { this.rawPcmListeners.forEach((listener) => listener(bytes)); } + if (this.micLevelListeners.size > 0) { + const level = pcmLevel(bytes); + this.micLevelListeners.forEach((listener) => listener(level)); + } }, onFrameMeta: (angleDegrees: number, ssr: number) => { if (this.frameMetaListeners.size === 0) return; diff --git a/app/prompts.ts b/app/prompts.ts index 52b4de39..d5161cfd 100644 --- a/app/prompts.ts +++ b/app/prompts.ts @@ -85,15 +85,20 @@ export const TRANSLATE_SYSTEM_PROMPT = "Reply with the English translation only, with no preamble, quotes, notes, or transliteration. " + "If the speech is empty or meaningless noise, reply with nothing."; -/** User message for the Translate app: earlier turns for context, then the speech to translate. */ +/** + * User message for the Translate app: earlier turns for context, then the + * speech to translate. `language` names the spoken language when the user set + * one (the transcription was told to expect it), else "". + */ export function buildTranslateUserMessage( speech: string, context: ReadonlyArray<{ source: string; translation: string }>, + language = "", ): string { const earlier = context.length ? "Earlier in this conversation (already translated; context only):\n" + context.map((turn) => `${turn.source}\n${turn.translation}`).join("\n") + "\n\n" : ""; - return `${earlier}Translate this speech into English:\n${speech}`; + return `${earlier}Translate this ${language ? `${language} ` : ""}speech into English:\n${speech}`; } diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 3b98c2d0..4101810c 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -14,14 +14,16 @@ export type ContinuousCaptureOptions = { /** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */ recordAs?: string; /** - * Transcribe with this provider instead of the Transcription Provider - * setting, letting it detect the spoken language (Translate). While any - * client asks for one, every capture uses it, and a shared capture running - * another provider restarts on it. + * Transcribe with this provider, expecting this spoken language (ISO 639-1, + * or "" for the provider to detect it), instead of the Transcription + * Provider setting (Translate). While any client asks for one, every + * capture uses it, and a shared capture running another restarts on it. */ - multilingualProvider?: VoiceProvider; + transcription?: TranscriptionChoice; }; +export type TranscriptionChoice = { provider: VoiceProvider; language: string }; + export type LayerActions = { /** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */ requestRender: () => void; diff --git a/app/ui/listening-header.ts b/app/ui/listening-header.ts new file mode 100644 index 00000000..a93a2370 --- /dev/null +++ b/app/ui/listening-header.ts @@ -0,0 +1,92 @@ +import { type GrayImage, type UiFont } from "../graphics/image"; +import { truncateText } from "../graphics/textwrap"; +import { voiceControlBridge } from "../native/voice-control"; +import { lineStep } from "./metrics"; + +const HEADER_X = 24; +const HEADER_Y = 20; +const TITLE_GRAY = 200; +const STATUS_GRAY = 110; + +const SEGMENTS = 8; +const SEGMENT_WIDTH = 4; +const SEGMENT_GAP = 2; +const METER_GAP = 10; +const LIT_GRAY = 200; +const UNLIT_GRAY = 45; +/** How often the meter may ask for a repaint (each one is a frame over BLE). */ +const TICK_MS = 200; +/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */ +const DECAY = 0.5; + +/** + * A compact segmented mic-level meter for listening views. Levels arrive per + * 50 ms audio chunk; the meter holds their peak and, at most every 200 ms, + * asks for a repaint only when the number of lit segments changed and the + * window is on screen. + */ +export class MicLevelMeter { + readonly width = SEGMENTS * (SEGMENT_WIDTH + SEGMENT_GAP) - SEGMENT_GAP; + private peak = 0; + private shown = 0; + private lit = 0; + private timer: ReturnType | null = null; + private unsubscribe: (() => void) | null = null; + + start(requestRender: () => void, isVisible: () => boolean): void { + this.unsubscribe = voiceControlBridge.onMicLevel((level) => { + this.peak = Math.max(this.peak, level); + }); + this.timer = setInterval(() => { + this.shown = Math.max(this.peak, this.shown * DECAY); + this.peak = 0; + const lit = Math.min(SEGMENTS, Math.round(this.shown * SEGMENTS)); + if (lit === this.lit) return; + this.lit = lit; + if (isVisible()) requestRender(); + }, TICK_MS); + } + + stop(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + if (this.timer !== null) clearInterval(this.timer); + this.timer = null; + } + + /** Draw the segments with their bottoms on `baseline`, `height` tall. */ + paint(image: GrayImage, x: number, baseline: number, height: number): void { + for (let index = 0; index < SEGMENTS; index++) { + const segmentX = x + index * (SEGMENT_WIDTH + SEGMENT_GAP); + image.fillRect(segmentX, baseline - height, SEGMENT_WIDTH, height, index < this.lit ? LIT_GRAY : UNLIT_GRAY); + } + } +} + +/** + * The one-line header of the listening views (Transcribe, Translate): + * "Title · status", the status truncated to fit, then the mic level meter at + * the line's end when one is given (while listening). Returns the y at which + * the view's body starts. + */ +export function paintListeningHeader( + image: GrayImage, + font: UiFont, + title: string, + status: string, + meter: MicLevelMeter | null, +): number { + let textRight = image.width - HEADER_X; + if (meter) { + const meterX = textRight - meter.width; + // Roughly cap height, sitting on the text's baseline. + meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6))); + textRight = meterX - METER_GAP; + } + image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY); + if (status) { + const statusX = HEADER_X + Math.ceil(font.measureText(title)); + image.drawText(font, statusX, HEADER_Y, truncateText(font, ` · ${status}`, textRight - statusX), STATUS_GRAY); + } + return HEADER_Y + lineStep(font) + 2; +} diff --git a/app/util/audio-level.ts b/app/util/audio-level.ts new file mode 100644 index 00000000..acfd9516 --- /dev/null +++ b/app/util/audio-level.ts @@ -0,0 +1,24 @@ +/** + * Mic level-meter math shared by the Microphones app's meters and the + * listening header (Transcribe, Translate): RMS mapped onto a 60 dB meter. + */ + +/** Meter position in [0,1] for a chunk of 16-bit little-endian PCM. */ +export function pcmLevel(pcm: Uint8Array): number { + let sumSquares = 0; + const count = pcm.length / 2; + if (count === 0) return 0; + for (let i = 0; i < count; i++) { + let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); + if (value >= 0x8000) value -= 0x10000; + sumSquares += value * value; + } + return rmsToLevel(Math.sqrt(sumSquares / count) / 32768); +} + +/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */ +export function rmsToLevel(rms: number): number { + if (rms <= 0) return 0; + const db = 20 * Math.log10(rms); + return Math.max(0, Math.min(1, (db + 60) / 60)); +} diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 2e394e2e..06abfb5f 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -2,7 +2,7 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { loader } = require('./helpers/load-typescript.cjs'); -function harness(provider = 'elevenlabs', continuous = true) { +function harness(provider = 'elevenlabs', continuous = true, extraOptions = {}) { const timers = new Map(); let timerId = 0; const sockets = []; @@ -30,7 +30,7 @@ function harness(provider = 'elevenlabs', continuous = true) { const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]]; const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts'); const client = new ReconnectingSttClient(options => new Provider(options), { - apiKey: 'test', onTranscript: event => events.push(event), + apiKey: 'test', ...extraOptions, onTranscript: event => events.push(event), onStatus: status => statuses.push(status), onError: error => errors.push(error), }, () => continuous); client.start(); @@ -203,7 +203,10 @@ test('provider selection falls back to on-device without a key and connects the FaceclawWebSocket: class { constructor(url, _listener, header, value) { sockets.push({ url, header, value }); } }, } } }, }, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } }); - const { createCloudSttClient, usesCloudStt } = load('app/native/cloud-stt-provider.ts'); + const { createCloudSttClient, usesCloudStt, transcriptionKey } = load('app/native/cloud-stt-provider.ts'); + // A running capture is shared only on the same provider and language. + assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja'); + assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); const statuses = []; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; @@ -217,4 +220,23 @@ test('provider selection falls back to on-device without a key and connects the createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: ' xi ' }, options, () => false).start(); assert.deepEqual(sockets.map(({ url, header, value }) => [new URL(url).host, header, value]), [['stt-rt.soniox.com', null, null], ['api.elevenlabs.io', 'xi-api-key', 'xi']]); + createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: 'xi', language: ' ja ' }, options, () => false).start(); + assert.equal(new URL(sockets.at(-1).url).searchParams.get('language_code'), 'ja'); +}); + +test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => { + for (const language of ['ja', '']) { + const elevenlabs = harness('elevenlabs', true, { language }); + assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null); + + const soniox = harness('soniox', true, { language }); + soniox.open(); + assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined); + + const openai = harness('openai', true, { language }); + openai.open(); + const transcription = JSON.parse(openai.sockets[0].sent[0]).session.audio.input.transcription; + assert.equal(transcription.model, 'gpt-realtime-whisper'); + assert.equal(transcription.language, language || undefined); + } }); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 8631bb16..ef05adf7 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -348,7 +348,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec get continuousActive() { return !!this.source || this.draining; }, prepare: async () => true, startGlassesCapture: async (_session, voice, _log, _endpointing, continuous) => { - calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.whisperServerLanguage}`); bridge.source = 'glasses'; }, + calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.language}`); bridge.source = 'glasses'; }, startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; }, drain() { if (this.source) { this.source = null; this.draining = true; } }, stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); }, @@ -361,7 +361,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings : id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } - : id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.whisperServerLanguage}` } : {}, + : id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.language}` } : {}, }); const host = Object.create(IosPreviewController.prototype); Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true, @@ -381,14 +381,17 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec host.stopContinuousVoiceCapture('calculator'); assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening'); - // Translate needs a multilingual provider: the shared capture drains and - // restarts on it (Whisper left to detect the language), and moves back to - // the main provider once Translate closes. - calls.length = 0; host.startContinuousVoiceCapture('translate', { multilingualProvider: 'soniox' }); await settle(); + // Translate needs its own provider and language: the shared capture drains + // and restarts on them, again when only the language changes, and moves + // back to the main provider (and Whisper's own language) once Translate closes. + calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: '' } }); await settle(); assert.deepEqual(calls, [['stop']]); drained(); await settle(); assert.deepEqual(calls, [['stop'], ['glasses', true]]); assert.deepEqual(providers.slice(-2), ['whisper-server:en', 'soniox:']); + calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: 'ja' } }); drained(); await settle(); + assert.deepEqual(calls, [['stop'], ['glasses', true]]); + assert.equal(providers.at(-1), 'soniox:ja'); calls.length = 0; host.stopContinuousVoiceCapture('translate'); drained(); await settle(); assert.deepEqual(calls, [['stop'], ['glasses', true]]); assert.equal(providers.at(-1), 'whisper-server:en'); diff --git a/tests/listening-header.test.cjs b/tests/listening-header.test.cjs new file mode 100644 index 00000000..19ffc044 --- /dev/null +++ b/tests/listening-header.test.cjs @@ -0,0 +1,100 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +const { pcmLevel } = loader()('app/util/audio-level.ts'); + +function harness() { + const levelListeners = new Set(); + let tick = null; + const { MicLevelMeter, paintListeningHeader } = loader({ + setInterval(fn) { tick = fn; return 1; }, + clearInterval() { tick = null; }, + }, { + '../native/voice-control': { voiceControlBridge: { + onMicLevel(listener) { levelListeners.add(listener); return () => levelListeners.delete(listener); }, + } }, + })('app/ui/listening-header.ts'); + const level = value => { for (const listener of levelListeners) listener(value); }; + return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners }; +} + +/** Records draws; text is 6px per character, like a fixed-width bitmap face. */ +function fakeImage(width = 576) { + const texts = [], rects = []; + return { width, texts, rects, + drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); }, + fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } }; +} +const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 }; + +function pcm(amplitude, samples = 800) { + const bytes = new Uint8Array(samples * 2); + for (let i = 0; i < samples; i++) { + const value = (i % 2 ? amplitude : -amplitude) & 0xffff; + bytes[i * 2] = value & 255; + bytes[i * 2 + 1] = value >> 8; + } + return bytes; +} + +test('pcm level maps RMS onto a 60 dB meter', () => { + assert.equal(pcmLevel(new Uint8Array(0)), 0); + assert.equal(pcmLevel(pcm(0)), 0); + assert.ok(Math.abs(pcmLevel(pcm(3277)) - 2 / 3) < 0.01, '-20 dBFS sits two thirds up'); + assert.ok(pcmLevel(pcm(32767)) > 0.99); +}); + +test('the meter repaints only when the lit segment count changes, and only while visible', () => { + const h = harness(); + const meter = new h.MicLevelMeter(); + let renders = 0, visible = true; + meter.start(() => renders++, () => visible); + h.level(0.3); h.level(0.5); h.level(0.2); + h.tick(); + assert.equal(renders, 1, 'the peak of the tick lights 4 of 8'); + h.level(0.5); h.tick(); + assert.equal(renders, 1, 'same segment count, no repaint'); + + // Off screen the level still tracks, so the next visible paint is current. + visible = false; + h.level(1); h.tick(); + assert.equal(renders, 1); + const image = fakeImage(); + meter.paint(image, 0, 10, 6); + assert.equal(image.rects.filter(rect => rect.gray === 200).length, 8); + + // Silence decays the meter to empty within about a second. + visible = true; + for (let i = 0; i < 5; i++) h.tick(); + const after = fakeImage(); + meter.paint(after, 0, 10, 6); + assert.equal(after.rects.filter(rect => rect.gray === 200).length, 0); + assert.ok(renders > 1); + + meter.stop(); + assert.equal(h.ticking(), false); + assert.equal(h.levelListeners.size, 0); +}); + +test('the header puts title and status on one line, truncated before the meter', () => { + const h = harness(); + const meter = new h.MicLevelMeter(); + const image = fakeImage(300); + const bodyTop = h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter); + assert.equal(bodyTop, 20 + 18 + 2); + assert.deepEqual(image.texts.map(({ x, y, gray }) => [x, y, gray]), [[24, 20, 200], [24 + 54, 20, 110]]); + assert.equal(image.texts[0].text, 'Translate'); + const status = image.texts[1].text; + assert.ok(status.startsWith(' · Listening') && status.endsWith('...'), status); + const meterX = Math.min(...image.rects.map(rect => rect.x)); + assert.equal(meterX, 300 - 24 - meter.width); + assert.ok(24 + 54 + status.length * 6 <= meterX - 10, 'status stops short of the meter'); + // Segments sit on the text baseline. + assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent)); + + const idle = fakeImage(300); + h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null); + assert.equal(idle.rects.length, 0); + assert.equal(idle.texts[1].text, ' · Starting microphone...'); +}); diff --git a/tests/translate.test.cjs b/tests/translate.test.cjs index a2bdf3ca..c9446b15 100644 --- a/tests/translate.test.cjs +++ b/tests/translate.test.cjs @@ -32,14 +32,13 @@ function settingsHarness({ main = 'onboard', setUp = [] } = {}) { })('app/apps/translate/translate-settings.ts'); } -test('finals become segments; finals arriving mid-translation merge into one pending segment', () => { +test('finals translate in order; finals arriving mid-request merge into one waiting block', () => { const model = new TranslationModel(); model.accept({ text: 'Hola, ¿qué', isFinal: false }); - assert.equal(model.live, 'Hola, ¿qué'); model.accept({ text: 'Hola, ¿qué tal?', isFinal: true }); - assert.equal(model.live, ''); + assert.equal(model.segments.length, 1); const first = model.nextJob(); - assert.equal(first.segment.source, 'Hola, ¿qué tal?'); + assert.equal(first.source, 'Hola, ¿qué tal?'); assert.deepEqual(plain(first.context), []); assert.equal(model.nextJob(), null, 'one request at a time'); @@ -49,30 +48,91 @@ test('finals become segments; finals arriving mid-translation merge into one pen assert.equal(model.segments.length, 2); assert.equal(model.segments[1].source, 'Muy bien. Gracias.'); - model.progress(first.segment, 'Hi, how'); - assert.equal(first.segment.translation, 'Hi, how'); - model.complete(first.segment, 'Hi, how are you?'); + assert.equal(model.progress(first, 'Hi, how'), true); + assert.equal(first.segment.streaming, 'Hi, how'); + model.complete(first, 'Hi, how are you?'); const second = model.nextJob(); - assert.equal(second.segment.source, 'Muy bien. Gracias.'); + assert.equal(second.source, 'Muy bien. Gracias.'); assert.deepEqual(plain(second.context), [{ source: 'Hola, ¿qué tal?', translation: 'Hi, how are you?' }]); - model.complete(second.segment, 'Very well. Thanks.'); + model.complete(second, 'Very well. Thanks.'); assert.equal(model.nextJob(), null); }); +test('the live block is translated provisionally, redone as it grows, and settles on a matching final', () => { + const model = new TranslationModel(); + model.accept({ text: '今日は', isFinal: false }); + const first = model.nextJob(); + assert.equal(first.source, '今日は'); + model.accept({ text: '今日はいい天気', isFinal: false }); // grows while the request is in flight + assert.equal(model.nextJob(), null); + model.complete(first, 'Today'); + assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'Today', gray: 170 }]); + + const second = model.nextJob(); + assert.equal(second.source, '今日はいい天気'); + // The redo streams over the old text only once it is as long. + model.progress(second, 'To'); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Today']); + model.progress(second, 'The weather'); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['The weather']); + model.complete(second, 'The weather is nice today'); + + // A final matching what was translated needs no further request. + model.accept({ text: '今日はいい天気', isFinal: true }); + assert.equal(model.nextJob(), null); + assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'The weather is nice today', gray: 230 }]); +}); + +test('a final that differs from the request in flight makes it moot; a matching one keeps it', () => { + const model = new TranslationModel(); + model.accept({ text: 'Bonjour', isFinal: false }); + const stale = model.nextJob(); + assert.equal(model.accept({ text: 'Bonjour à tous', isFinal: true }), true); + assert.equal(model.isActive(stale), false); + assert.equal(model.progress(stale, 'Hello'), false); + const redo = model.nextJob(); + assert.equal(redo.source, 'Bonjour à tous'); + model.complete(stale, 'Hello'); // a late reply from the cancelled request is ignored + assert.equal(model.isActive(redo), true); + model.complete(redo, 'Hello everyone'); + + model.accept({ text: 'Merci', isFinal: false }); + const kept = model.nextJob(); + assert.equal(model.accept({ text: 'Merci', isFinal: true }), false); + model.complete(kept, 'Thank you'); + assert.equal(model.nextJob(), null); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.gray)), [230, 230]); +}); + +test('small revisions of a live block wait for a bigger change; its final is always redone', () => { + const model = new TranslationModel(); + model.accept({ text: 'abcdef', isFinal: false }); + model.complete(model.nextJob(), 'one'); + model.accept({ text: 'abcdeg', isFinal: false }); + assert.equal(model.nextJob(), null); + model.accept({ text: 'abcdefgh', isFinal: false }); + assert.equal(model.nextJob(), null); + model.accept({ text: 'abcdefghi', isFinal: false }); + model.complete(model.nextJob(), 'two'); + model.accept({ text: 'abcdefghj', isFinal: true }); + assert.equal(model.nextJob().source, 'abcdefghj'); +}); + test('an empty final keeps the live partial it replaces', () => { const model = new TranslationModel(); model.accept({ text: 'Bonjour', isFinal: false }); model.accept({ text: '', isFinal: true }); assert.equal(model.segments[0].source, 'Bonjour'); + assert.equal(model.segments[0].final, true); }); -test('context skips failed and dropped turns and stays bounded', () => { +test('context skips failed, dropped and provisional turns and stays bounded', () => { const model = new TranslationModel(); const translate = (source, translation) => { model.accept({ text: source, isFinal: true }); const job = model.nextJob(); - if (translation === null) model.fail(job.segment); - else model.complete(job.segment, translation); + if (translation === null) model.fail(job); + else model.complete(job, translation); }; translate('uno', 'one'); translate('ruido', null); @@ -87,18 +147,29 @@ test('context skips failed and dropped turns and stays bounded', () => { const long = new TranslationModel(); for (let i = 0; i < 3; i++) { long.accept({ text: 'x'.repeat(500), isFinal: true }); - long.complete(long.nextJob().segment, 'y'.repeat(500)); + long.complete(long.nextJob(), 'y'.repeat(500)); } long.accept({ text: 'next', isFinal: true }); assert.equal(long.nextJob().context.length, 1, 'the newest turn is kept even when it alone is over the cap'); + + // A final whose redo failed still shows its provisional translation, but + // that translates other text, so it isn't context. + const stale = new TranslationModel(); + stale.accept({ text: 'Ich gehe', isFinal: false }); + stale.complete(stale.nextJob(), 'I go'); + stale.accept({ text: 'Ich gehe nach Hause', isFinal: true }); + stale.fail(stale.nextJob()); + assert.deepEqual(plain(stale.lines(measure, 100, false)), [{ text: 'I go', gray: 170 }]); + stale.accept({ text: 'Tschüss', isFinal: true }); + assert.deepEqual(plain(stale.nextJob().context), []); }); -test('display lines: translations bright, originals dim until translated or when shown, live dimmest', () => { +test('display lines: settled bright, provisional dimmer, originals dim until translated or when shown', () => { const model = new TranslationModel(); model.accept({ text: 'Guten Morgen', isFinal: true }); - model.complete(model.nextJob().segment, 'Good morning'); + model.complete(model.nextJob(), 'Good morning'); model.accept({ text: 'Wie geht es?', isFinal: true }); - const asking = model.nextJob().segment; + const asking = model.nextJob(); model.accept({ text: 'Danke', isFinal: false }); assert.deepEqual(plain(model.lines(measure, 100, false)), [ { text: 'Good morning', gray: 230 }, @@ -107,28 +178,29 @@ test('display lines: translations bright, originals dim until translated or when ]); model.progress(asking, 'How are'); assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Good morning', 'How are', 'Danke']); - assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 230, 90]); + assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 170, 90]); assert.deepEqual(plain(model.lines(measure, 7, false).slice(0, 2)), [{ text: 'Good', gray: 230 }, { text: 'morning', gray: 230 }]); - // A failed segment keeps its original on screen. + // A failed block keeps its original on screen. model.fail(asking); assert.deepEqual(plain(model.lines(measure, 100, false)[1]), { text: 'Wie geht es?', gray: 120 }); - // A finished, empty translation is noise: hidden unless originals are shown. + // A settled, empty translation is noise: hidden unless originals are shown. model.accept({ text: 'mmm', isFinal: true }); - model.complete(model.nextJob().segment, ''); + model.complete(model.nextJob(), ''); assert.equal(model.lines(measure, 100, false).some(line => line.text === 'mmm'), false); assert.equal(model.lines(measure, 100, true).some(line => line.text === 'mmm'), true); }); -test('saved text pairs each original with its translation', () => { +test('saved text pairs each final original with its translation', () => { const model = new TranslationModel(); for (const [source, translation] of [['Ciao', 'Hello'], ['Okay.', 'Okay.'], ['boh', null]]) { model.accept({ text: source, isFinal: true }); const job = model.nextJob(); - if (translation === null) model.fail(job.segment); - else model.complete(job.segment, translation); + if (translation === null) model.fail(job); + else model.complete(job, translation); } + model.accept({ text: 'ancora', isFinal: false }); // still live: not saved assert.equal(model.savedText(), 'Ciao\n→ Hello\n\nOkay.\n\nboh\n→ (not translated)'); }); @@ -138,6 +210,8 @@ test('the user message carries earlier turns as context, then the speech', () => assert.match(message, /context only/); assert.match(message, /Bien<\/speech>\nFine<\/english>\n\nTranslate this speech/); assert.ok(message.endsWith('¿Y tú?')); + assert.equal(buildTranslateUserMessage('こんにちは', [], 'Japanese'), + 'Translate this Japanese speech into English:\nこんにちは'); }); test('Auto picks the main provider when it is multilingual, else the first one set up', () => { @@ -164,3 +238,15 @@ test('an explicit provider is used only while it is set up, and on-device ones a assert.equal(settings.translateProviderSetting.isDisabled('auto'), false); assert.deepEqual(plain(settings.translateProviderSetting.values), ['auto', 'elevenlabs', 'whisper', 'soniox', 'whisper-server']); }); + +test('the spoken language defaults to auto-detect and resolves to an ISO 639-1 code once chosen', () => { + const settings = settingsHarness(); + assert.equal(settings.resolveTranslateLanguage(), ''); + assert.equal(settings.translateLanguageSetting.displayValue('auto'), 'Auto-detect'); + settings.translateLanguageSetting.set('ja'); + assert.equal(settings.resolveTranslateLanguage(), 'ja'); + assert.equal(settings.spokenLanguageName('ja'), 'Japanese'); + assert.equal(settings.translateLanguageSetting.values[0], 'auto'); + assert.ok(settings.translateLanguageSetting.values.slice(1).every(code => /^[a-z]{2}$/.test(code))); + assert.equal(settings.translateLanguageSetting.values.includes('en'), false, 'English is the target, not a source'); +}); From 281b6ac53fa029863b7ad20db5d4e7a47ac38e8e Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 13:18:07 -0700 Subject: [PATCH 03/34] Update CHANGELOG --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 8bbbd05c..5880581f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,8 @@ * Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover * Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view * Add an Apple Music integration on iOS +* Add a Spotify integration on iOS +* Add a Translate app ### Bug fixes From 99f146529b7cb9773a5934ab7e604f1c993b27ba Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 13:42:28 -0700 Subject: [PATCH 04/34] Fix missing ring battery indicator on iOS --- CHANGELOG | 1 + app/g2/ios-preview-controller.ts | 23 +++++++++++++++++------ tests/ios-glanceboard.test.cjs | 26 +++++++++++++++++++++----- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2750fbfb..f61d5fcb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ * Inverting Android notification-icons to indicate selectionis now gamma-correct * Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks +* Fix missing R1 ring battery indicator on iOS ## 0.8.4 - 2026-10-17 diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index b3fe04a5..d56ba906 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -65,7 +65,7 @@ const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 2000 export type SessionPhase = 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error' export type SessionState = { phase: SessionPhase; status: string; battery: number | null; charging: boolean | null; - ring: boolean; frames: number; capabilities: string; leftVersion: string; rightVersion: string; + ring: boolean; ringBattery: number | null; ringCharging: boolean | null; frames: number; capabilities: string; leftVersion: string; rightVersion: string; /** Why the connected firmware can't run Faceclaw (firmware-compat.ts), or '' when it can or is unknown. */ firmwareWarning: string } @@ -84,7 +84,7 @@ export class IosPreviewController { private communicator: FaceclawCommunicatorBridge | null = null private readonly startedCommunicators = new WeakSet() state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false, - frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' } + ringBattery: null, ringCharging: null, frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' } private notifications: AncsClient | null = null private readonly sessionOffs: (() => void)[] = [] private stopping: Promise | null = null @@ -741,12 +741,18 @@ export class IosPreviewController { if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() this.syncContinuousSession() - shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging }) + this.pushBatteryLevels() this.syncPresence() this.syncRuntime() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() } + /** The ring level arrives through the glasses, so like Android it is shown only while connected. */ + private pushBatteryLevels(): void { + const connected = this.state.phase === 'connected' + shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging, + ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null }) + } async connect(): Promise { if (!this.active || this.connecting || this.stopping) return if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return @@ -755,7 +761,8 @@ export class IosPreviewController { this.connecting = true this.sessionCreated = true try { - this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false } + this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false, + ringBattery: null, ringCharging: null } this.firmwareCompatible = false this.update('connecting', 'Finding configured devices…') let identifiers @@ -803,10 +810,14 @@ export class IosPreviewController { communicator.onRingEvent(input => this.handleRingEvent(input)), communicator.onBatteryState(state => { if (this.communicator !== communicator) return + // Unlike the headset level, an unavailable ring reading (-1: ring away or old + // firmware) clears the indicator, as on Android. + const ringBattery = state.ringBattery ?? -1, hasRingBattery = Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100 this.state = { ...this.state, battery: state.battery >= 0 && state.battery <= 100 ? state.battery : this.state.battery, charging: state.chargingStatus >= 0 ? state.chargingStatus > 0 : this.state.charging, - ring: (state.ringBattery ?? -1) >= 0 || this.state.ring } - shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging }) + ring: hasRingBattery || this.state.ring, + ringBattery: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null } + this.pushBatteryLevels() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() }), diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index fb377690..f1dd8c63 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -25,6 +25,7 @@ function fixture() { const observers = new Map(), phoneState = { protectedDataAvailable: true }; let settingsChanged, remoteHost, controller, bridge = null; let now = 1000, nextTask = 0, screenOn = true, shellOptions; + const battery = {}; const tasks = new Map(), sent = [], previews = [], received = [], errors = []; const boardStats = { starts: 0, stops: 0, paints: 0 }; const settings = { lock: true, enabled: true, tap: true, hold: true, tilt: true, duration: 3000, depth: 0 }; @@ -47,7 +48,7 @@ function fixture() { wake: () => { screenOn = true; shellOptions.onScreenStateChanged(true); }, sleep: () => { screenOn = false; shellOptions.onScreenStateChanged(false); }, isScreenOn: () => screenOn, getWindows: () => [window], foregroundWindow: () => window, - setBatteryLevels() {}, underlayDim: () => 1, getFocus: () => 'app', hasOverlay: () => false, + setBatteryLevels: levels => Object.assign(battery, levels), underlayDim: () => 1, getFocus: () => 'app', hasOverlay: () => false, describeInputTarget: () => 'test app', paintScene: () => new Uint8Array([0, 0]), paintSurface: () => [{ image: new images.GrayImage(640, 480, 0), x: 0, y: 0 }], receiveInput: async input => { @@ -105,13 +106,14 @@ function fixture() { constructor() { super(); bridge = this; } on(kind, fn) { this.listeners[kind] = fn; return () => { if (this.listeners[kind] === fn) delete this.listeners[kind]; }; } onStateChange(fn) { return this.on('state', fn); } onRingEvent(fn) { return this.on('ring', fn); } - onWearState(fn) { return this.on('wear', fn); } onBatteryState() { return noop(); } onFirmwareInfo() { return noop(); } - onFrameMetrics() { return noop(); } addCompassListener() { return noop(); } onAncsRelayFrame() { return noop(); } + onWearState(fn) { return this.on('wear', fn); } onBatteryState(fn) { return this.on('battery', fn); } onFirmwareInfo() { return noop(); } + onFrameMetrics() { return noop(); } onPreviewAnimationFrame() { return noop(); } addCompassListener() { return noop(); } onAncsRelayFrame() { return noop(); } onAncsAuthorization() { return noop(); } setRequiresAncs() {} rightWriteLimit() { return 20; } onPhoneLockSignal() {} async writeRawToRight() {} async configureBrightness() {} async setBrightness() {} async enableWearDetectionAndRequestState() {} async playBuzzerSequence() {} async close() {} emitState(phase, status = phase) { if (phase === 'connected' || phase === 'disconnected') this.phase = phase; this.listeners.state?.({ phase, status }); } emitRing(input) { this.listeners.ring?.(input); } emitWear(wearing) { this.listeners.wear?.(wearing); } + emitBattery(state) { this.listeners.battery?.(state); } async start() { this.emitState('connected', 'Connected.'); } async disconnect() { this.emitState('disconnected', 'Disconnected.'); } waitForFrameFinished() { return Promise.resolve('sent'); } @@ -160,7 +162,7 @@ function fixture() { '../apps/all-apps': { ALL_APPS: [{ appId: 'glanceboard', glanceboard: provider }] }, '../apps/onboarding/onboarding-progress': { claimGlassesOnboardingAutoLaunch: () => false }, '../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {}, - '../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, getBrightnessPreferences: () => ({ auto: true, level: 50, minimum: 2, maximum: 100, curve: '0:0,1000:100', fadeMs: 280 }), lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } }, + '../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, getBrightnessPreferences: () => ({ auto: true, level: 50, minimum: 2, maximum: 100, curve: '0:0,1000:100', fadeMs: 280 }), lockScreenEnabledSetting: { get: () => settings.lock }, ringConnectionModeSetting: { get: () => 'glasses' }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } }, '../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) }, '../apps/ios-availability': { iosAppUnavailableReason: () => null }, '../graphics/glyph-wire': { prepareFrameDraws: () => null }, @@ -202,7 +204,7 @@ function fixture() { } async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); } async function connect() { await controller.connect(); await render(); } - return { controller, shell, remoteHost, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, + return { controller, shell, battery, remoteHost, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, get bridge() { return bridge; }, connect, wear: wearing => bridge.emitWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render }; } const assertBlank = pixels => assert.ok(pixels.every(p => p === 0)); @@ -242,3 +244,17 @@ test('keyboard sessions cannot deliver text after the glasses lock or phone leav editor.discard(); assert.deepEqual(calls.at(-1), ['discard']); }); + +test('ring battery reported through the glasses reaches the shell while connected', async () => { + const f = fixture(); + await f.connect(); + f.bridge.emitBattery({ battery: 70, chargingStatus: 0, ringBattery: 55, ringChargingStatus: 1 }); + assert.deepEqual([f.battery.headset, f.battery.ring, f.battery.ringCharging], [70, 55, true]); + // An unavailable ring reading clears the indicator; the headset keeps its last level. + f.bridge.emitBattery({ battery: -1, chargingStatus: 0, ringBattery: -1, ringChargingStatus: -1 }); + assert.deepEqual([f.battery.headset, f.battery.ring, f.battery.ringCharging], [70, null, null]); + f.bridge.emitBattery({ battery: 70, chargingStatus: 0, ringBattery: 40, ringChargingStatus: 0 }); + assert.deepEqual([f.battery.ring, f.battery.ringCharging], [40, false]); + await f.controller.disconnect(); await f.render(); + assert.deepEqual([f.battery.ring, f.battery.ringCharging], [null, null]); +}); From 247459bf993daeb6e461b521f45c81558a54710e Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 14:04:48 -0700 Subject: [PATCH 05/34] Fix an issue that could cause corrupted installs of EvenHub apps, and redirect corrupted installs into the update/reinstall --- CHANGELOG | 1 + app/apps/evenhub/installed-apps.ts | 15 ++++++++++++++ app/g2/ios-preview-controller.ts | 14 +++++++++---- app/native/binary-http.ios.ts | 11 ++++++++-- app/native/voice-control.ios.ts | 3 ++- tests/ios-app-host.test.cjs | 24 ++++++++++++++++++++++ tests/ios-evenhub.test.cjs | 32 ++++++++++++++++++++++++++++++ 7 files changed, 93 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f61d5fcb..2badb3b3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,6 +15,7 @@ * Inverting Android notification-icons to indicate selectionis now gamma-correct * Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks * Fix missing R1 ring battery indicator on iOS +* Fix a bug on iOS that would cause corrupted installs of EvenHub apps ## 0.8.4 - 2026-10-17 diff --git a/app/apps/evenhub/installed-apps.ts b/app/apps/evenhub/installed-apps.ts index a785f34f..bf6dfc43 100644 --- a/app/apps/evenhub/installed-apps.ts +++ b/app/apps/evenhub/installed-apps.ts @@ -176,6 +176,21 @@ export function readEvenHubPackageManifestBytes(bytes: Uint8Array): EvenHubManif } } +/** + * Delete an installed EHPK that is all zeros, keeping its registry entry and + * icon so the store treats it as missing and offers Reinstall. iOS builds + * before the binary-http.ios.ts copy fix could store a large download that + * way. Returns whether the package was discarded. + */ +export function discardZeroedEvenHubPackage(packageId: string): boolean { + const path = installedEvenHubPackagePath(packageId); + const bytes = readBinaryFile(path); + if (!bytes || bytes.length === 0) return false; + for (let i = 0; i < bytes.length; i++) if (bytes[i] !== 0) return false; + deletePathRecursively(path); + return true; +} + /** Remove the stored EHPK, unpacked runtime files, and registry entry. */ export function uninstallEvenHubPackage(packageId: string): boolean { const apps = getInstalledEvenHubApps(); diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index d56ba906..84af7403 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -4,7 +4,7 @@ import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup, readActiveNotifications } from '../native/notification-icons.ios' import { shouldShowNotificationOnGlasses } from '../native/notification-sources' import { launcherEntries } from '../apps/launcher' -import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps' +import { discardZeroedEvenHubPackage, getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps' import { isInstalledPackagePresent } from '../apps/evenhub/updates' import { openEvenHubStoreForPackage } from '../apps/evenhub' import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager' @@ -625,9 +625,15 @@ export class IosPreviewController { const host = ALL_APPS.find(candidate => candidate.appId === 'evenhub') if (installed && host) { try { - if (isInstalledPackagePresent(installed.packageId)) - await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed) - else await openEvenHubStoreForPackage(this.buildAppContext(host), installed) + if (isInstalledPackagePresent(installed.packageId)) { + try { await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed); return } + catch (error) { + // Checked only after a failure: it reads the whole package again. + if (!discardZeroedEvenHubPackage(installed.packageId)) throw error + console.log(`[ios-preview] Discarded zeroed package for ${installed.packageId}; opening store page`) + } + } + await openEvenHubStoreForPackage(this.buildAppContext(host), installed) } catch (error) { this.fail(error) } } diff --git a/app/native/binary-http.ios.ts b/app/native/binary-http.ios.ts index e65172ab..80eb7c1d 100644 --- a/app/native/binary-http.ios.ts +++ b/app/native/binary-http.ios.ts @@ -2,11 +2,18 @@ import { Http } from '@nativescript/core'; import { withUserAgent } from '../util/http'; import type { BinaryPostResponse, BinaryResponse } from './binary-http'; -/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */ +/** + * Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. + * + * toArrayBuffer() is interop.bufferFromData, which in @nativescript/ios 9.0.3 + * views the NSData's bytes without retaining it: once the response is + * collected, the buffer is dangling (a store EHPK was saved as 41 MB of + * zeros). Copy while the response is still alive. + */ export async function downloadBinary(url: string, timeoutMs: number): Promise { const response = await Http.request({ url, method: 'GET', headers: withUserAgent(), timeout: timeoutMs }); const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1]; - return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) }; + return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()).slice(), contentLength: Number(length ?? 0) }; } /** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */ diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index f5a66124..4730ff81 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -70,7 +70,8 @@ export class IosVoiceControlBridge { this.native = FaceclawSpeech.new() this.native.eventHandler = (json: string) => this.receive(JSON.parse(json)) this.native.pcmHandler = (pcm: NSData) => { - const bytes = new Uint8Array(interop.bufferFromData(pcm)) + // Copied: bufferFromData doesn't retain pcm, and STT clients queue chunks. + const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice() this.cloud?.acceptPcm(bytes) if (this.continuous) this.record(bytes) if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) } diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index ef05adf7..37f29384 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -71,6 +71,30 @@ test('iOS installed apps launch, recover missing packages, and close before unin assert.deepEqual(calls, [['launch', 'installed', 'test.app'], ['reinstall', 'test.app'], ['close', 'test.app'], ['uninstall', 'test.app']]); }); +test('iOS launch failures on a zeroed package fall back to the store page; other failures still report', async () => { + const calls = [], failures = [], app = { packageId: 'test.app', name: 'Test' }; + let zeroed = true; + const modules = { + "../ui/input-monitor": load("app/ui/input-monitor.ts", {}), + '../apps/all-apps': { ALL_APPS: [{ appId: 'evenhub' }] }, + '../apps/evenhub/installed-apps': { getInstalledEvenHubAppById: () => app, + discardZeroedEvenHubPackage: id => { calls.push(['discard', id]); return zeroed; } }, + '../apps/evenhub/updates': { isInstalledPackagePresent: () => true }, + '../apps/evenhub': { openEvenHubStoreForPackage: (_ctx, value) => calls.push(['reinstall', value.packageId]) }, + '../apps/evenhub/manager': { launchInstalledPackage: async () => { calls.push(['launch']); throw new Error('Could not read the EHPK manifest.'); } }, + }; + const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => modules[id] ?? {} }); + const host = Object.create(IosPreviewController.prototype); + host.buildAppContext = app => app; host.fail = error => failures.push(error.message); host.requestShellRender = () => {}; + await host.launchApp('installed'); + assert.deepEqual(calls, [['launch'], ['discard', 'test.app'], ['reinstall', 'test.app']]); + assert.deepEqual(failures, []); + calls.length = 0; zeroed = false; + await host.launchApp('installed'); + assert.deepEqual(calls, [['launch'], ['discard', 'test.app']]); + assert.deepEqual(failures, ['Could not read the EHPK manifest.']); +}); + test('iOS settings prompts honor password input and report cancellation', async () => { const prompts = [], writes = []; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '@nativescript/core' ? { diff --git a/tests/ios-evenhub.test.cjs b/tests/ios-evenhub.test.cjs index 45d7fe99..7e9921f3 100644 --- a/tests/ios-evenhub.test.cjs +++ b/tests/ios-evenhub.test.cjs @@ -297,3 +297,35 @@ test('EvenHub assistant tools skip malformed names and schemas with a warning in assert.match(warnings[4], /"union": parameters can't use oneOf, anyOf or allOf/); h.session.close(); }); + +test('iOS binary downloads copy out of the response NSData, which bufferFromData does not retain', async () => { + const native = new Uint8Array([0x45, 0x48, 0x50, 0x4b, 1, 2, 3]); + const { downloadBinary } = loader({}, { + '@nativescript/core': { Http: { request: async () => ({ statusCode: 200, headers: {}, content: { toArrayBuffer: () => native.buffer } }) } }, + '../util/http': { withUserAgent: headers => headers ?? {} }, + })('app/native/binary-http.ios.ts'); + const { bytes } = await downloadBinary('https://cdn.example/app.ehpk', 1000); + native.fill(0); // The released NSData's pages are reused or zeroed. + assert.deepEqual(Array.from(bytes), [0x45, 0x48, 0x50, 0x4b, 1, 2, 3]); +}); + +test('zeroed installed packages are discarded with their registry entry kept; real packages are left alone', () => { + const files = new Map(), deleted = []; + const installed = loader({}, { + fzstd: require('fzstd'), + '../../native/file-access': { readBinaryFile: path => files.get(path) ?? null, readTextFile() {}, writeBinaryFile() {}, + appFilesDirPath: () => '/app', deletePathRecursively: path => { deleted.push(path); files.delete(path); } }, + '../../native/settings-store': { getStringSetting: (_key, fallback) => fallback, setStringSetting() {} }, + '../../native/icon-image': {}, '../../graphics/icons': {}, '../../graphics/image': {}, + })('app/apps/evenhub/installed-apps.ts'); + const path = installed.installedEvenHubPackagePath('test.app'); + files.set(path, pack({ 'app.json': '{"package_id":"test.app"}' })); + assert.equal(installed.discardZeroedEvenHubPackage('test.app'), false); + files.set(path, new Uint8Array([...new Uint8Array(1000), 1])); + assert.equal(installed.discardZeroedEvenHubPackage('test.app'), false); + assert.equal(installed.discardZeroedEvenHubPackage('missing.app'), false); + assert.deepEqual(deleted, []); + files.set(path, new Uint8Array(4096)); + assert.equal(installed.discardZeroedEvenHubPackage('test.app'), true); + assert.deepEqual(deleted, [path], 'only the EHPK goes; the icon and registry entry stay for Reinstall'); +}); From 53812609ed7fbb7d8042017fb7edf334e04031cf Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 14:19:45 -0700 Subject: [PATCH 06/34] Fix EvenHub apps on iOS access to the mic --- CHANGELOG | 2 + app/apps/evenhub/mic-router.ts | 1 + app/apps/evenhub/session.ts | 1 - app/g2/ios-preview-controller.ts | 14 ++++ app/native/voice-control.ios.ts | 123 ++++++++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2badb3b3..2a6d8960 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,8 @@ * Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks * Fix missing R1 ring battery indicator on iOS * Fix a bug on iOS that would cause corrupted installs of EvenHub apps +* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable") +* EvenHub apps on iOS should now be able to use the mic ## 0.8.4 - 2026-10-17 diff --git a/app/apps/evenhub/mic-router.ts b/app/apps/evenhub/mic-router.ts index 76f7fe15..bf7d7d57 100644 --- a/app/apps/evenhub/mic-router.ts +++ b/app/apps/evenhub/mic-router.ts @@ -27,6 +27,7 @@ export type EvenHubMicClient = { deliverAudioPcm(pcm: Uint8Array): void; }; +/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */ function activeCommunicator(): any { if (!global.isAndroid) return null; try { diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 53289885..bdbdd92d 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH * flows (foreground + screen on + no assistant modal). */ private audioControl(data: Record): boolean { - if (global.isIOS) return false; // EvenHub PCM routing is not ported yet. if (!permissionsIncludeMicrophone(this.manifest.permissions)) { this.log("evenhub: audioControl denied (app declares no microphone permission)"); return false; diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 84af7403..04ea57df 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -443,6 +443,7 @@ export class IosPreviewController { iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', 'Glasses locked. Transcription resumes when you unlock your phone.') } else this.syncContinuousCapture() + this.syncRawMicSession() void (async () => { await this.displayReady const display = this.display @@ -745,6 +746,8 @@ export class IosPreviewController { this.pushBrightness() this.maybeLaunchOnboarding(this.state) if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null + // Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session. + this.syncRawMicSession() if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() this.syncContinuousSession() this.pushBatteryLevels() @@ -952,6 +955,8 @@ export class IosPreviewController { for (const off of this.sessionOffs.splice(0)) off() this.notifications = null this.communicator = null + // Releases an EvenHub app's mic before the session closes. + this.syncRawMicSession() this.glance.reset() await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`)) } @@ -1065,6 +1070,15 @@ export class IosPreviewController { } if (current()) this.syncContinuousClaim() } + /** + * EvenHub apps' mic (the bridge's raw tap) listens through a connected, + * unlocked glasses session, as speech capture does; the EvenHub mic router + * decides which app hears it and when. + */ + private syncRawMicSession(): void { + const ready = this.state.phase === 'connected' && !this.glassesLocked + iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message)) + } /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ private syncContinuousClaim(): void { voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 4730ff81..2699515a 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -11,6 +11,9 @@ export type IosMicrophoneSession = { } /** How long a cloud provider gets, after the last audio, to send its final transcript. */ const CLOUD_FINAL_TIMEOUT_MS = 5000 +/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */ +const RAW_RETRY_MS = 3000 +const RAW_RETRY_LIMIT = 5 /** iOS implementation of the shared voice dialog's event bridge. Audio comes * from the glasses when connected, or the default phone input in preview mode, @@ -53,6 +56,21 @@ export class IosVoiceControlBridge { // and it is created by the first audio, so a failed start leaves no file. private recordingName = '' private recorder: WavRecorder | null = null + // Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to + // rawListeners while any owner wants it, from the session setRawSession + // bound. It shares the one glasses mic with speech capture, which preempts + // it; it resumes on its own once that capture has ended. + private rawNative: any = null + private readonly rawListeners = new Set<(pcm: Uint8Array) => void>() + private readonly rawOwners = new Set() + private rawSession: IosMicrophoneSession | null = null + // The session the tap enabled the mic on, while it is starting or live. + private rawHeld: IosMicrophoneSession | null = null + private rawLive = false + private rawGeneration = 0 + private rawFailures = 0 + private rawRetryTimer: ReturnType | null = null + private rawLog: (message: string) => void = () => {} onStatus(listener: (state: VoiceControlState) => void): () => void { this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } @@ -62,7 +80,7 @@ export class IosVoiceControlBridge { } onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } } onSpeechPause(_listener: () => void): () => void { return () => {} } - /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */ + /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */ onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } } private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) } private ensureNative(): any { @@ -102,6 +120,8 @@ export class IosVoiceControlBridge { get statusText(): string { return this.status } async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return + // Speech capture preempts the raw tap; its mic release is queued before this enable, which wins. + this.endRawTap() this.stop() const generation = ++this.generation this.continuous = continuous @@ -131,6 +151,7 @@ export class IosVoiceControlBridge { } async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return + this.endRawTap() this.stop() this.continuous = continuous this.log = log; this.source = 'phone'; this.capturing = true @@ -228,6 +249,100 @@ export class IosVoiceControlBridge { if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return } this.stop(); this.setStatus(message); this.emitEnd() } + /** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */ + onRawPcm(listener: (pcm: Uint8Array) => void): () => void { + this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) } + } + /** + * The glasses session the raw tap captures from (connected and unlocked), + * or null. Owners outlive a change: the tap moves to the next session. + */ + setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void { + this.rawLog = log + if (session === this.rawSession) return + this.endRawTap() + this.rawSession = session; this.rawFailures = 0 + this.syncRawTap() + } + /** + * Start the decode-only raw tap on behalf of `owner`; the communicator is + * the one setRawSession bound, not `options.communicator`. False while + * speech capture holds the mic or no session is bound: the owner stays + * registered and the tap starts once it can. Idempotent while running. + */ + startRawCapture(options: { communicator: any; owner: string }): boolean { + if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 } + this.syncRawTap() + return this.rawHeld !== null + } + /** `owner` no longer wants the raw tap; it stops once no owner does. */ + stopRawCapture(owner: string): void { + if (this.rawOwners.delete(owner)) this.syncRawTap() + } + /** Whether the raw tap's glasses mic enable went through and still holds. */ + isRawCaptureLive(): boolean { return this.rawLive } + /** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */ + private syncRawTap(): void { + const session = this.rawSession + if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return } + if (this.rawHeld || this.rawRetryTimer !== null) return + void this.beginRawTap(session) + } + private async beginRawTap(session: IosMicrophoneSession): Promise { + const generation = ++this.rawGeneration + const native = this.ensureRawNative() + const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '') + if (error) { this.failRawTap(error); return } + this.rawHeld = session + try { + await session.setMicrophone(true, packet => { + if (generation !== this.rawGeneration) return + const copy = new Uint8Array(packet) + native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) + }) + if (generation !== this.rawGeneration) return + this.rawLive = true + this.rawLog('EvenHub mic: glasses microphone enabled') + } catch (failure) { + if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure)) + } + } + private ensureRawNative(): any { + if (!this.rawNative) { + this.rawNative = FaceclawSpeech.new() + // A PCM capture only ends on its own when decoding failed or fell behind. + this.rawNative.eventHandler = (json: string) => { + const event = JSON.parse(json) + if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended')) + } + this.rawNative.pcmHandler = (pcm: NSData) => { + // Copied: bufferFromData doesn't retain pcm. + const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice() + // Audio is flowing, so a later failure is a new problem worth retrying. + this.rawFailures = 0 + for (const fn of [...this.rawListeners]) fn(bytes) + if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) } + } + } + return this.rawNative + } + /** Drop the tap and retry it shortly, a bounded number of times. */ + private failRawTap(message: string): void { + this.rawLog(`EvenHub mic: ${message}`) + this.endRawTap() + if (++this.rawFailures >= RAW_RETRY_LIMIT) return + this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS) + } + /** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */ + private endRawTap(): void { + ++this.rawGeneration + if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer) + this.rawRetryTimer = null + this.rawLive = false + this.rawNative?.cancel() + const session = this.rawHeld; this.rawHeld = null + if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`)) + } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } private releaseMicrophone(): void { const session = this.session; this.session = null @@ -261,7 +376,11 @@ export class IosVoiceControlBridge { // In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive. if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud() } - private emitEnd(): void { for (const fn of [...this.ends]) fn() } + private emitEnd(): void { + for (const fn of [...this.ends]) fn() + // After the listeners: one may start the next capture, which keeps the mic. + this.syncRawTap() + } private finishCompletion(): void { const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() } From 22f869d31c22d71abeda3281ffc947719b4f2578 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 14:20:39 -0700 Subject: [PATCH 07/34] notes/ LLM memories doesn't go in git --- notes/apps.txt | 87 ------ notes/hermes-assistant.md | 41 --- notes/ios-protocol-sync.md | 66 ---- notes/media-resume-via-media-key.md | 13 - notes/mentraos-compat-assessment.md | 133 -------- notes/voice-assistant-design.md | 466 ---------------------------- notes/whisper-server.md | 53 ---- 7 files changed, 859 deletions(-) delete mode 100644 notes/apps.txt delete mode 100644 notes/hermes-assistant.md delete mode 100644 notes/ios-protocol-sync.md delete mode 100644 notes/media-resume-via-media-key.md delete mode 100644 notes/mentraos-compat-assessment.md delete mode 100644 notes/voice-assistant-design.md delete mode 100644 notes/whisper-server.md diff --git a/notes/apps.txt b/notes/apps.txt deleted file mode 100644 index c6d66831..00000000 --- a/notes/apps.txt +++ /dev/null @@ -1,87 +0,0 @@ -# Even-Analogous Apps - -Ideally we'd like full feature parity with the stock Even app (or at least, -close enough that not-using the stock app doesn't feel like a sacrifice). -A rough list of things the stock app has that we currently don't: - -## Quicklist - -Quicklist is Even's version of a todo app, in the form of a dashboard card -that you add to and remove from with assistant voice commands. It's pretty -simple to build once a general voice assistant is in place, but I don't -actually like their design: a todo list really wants to be synchronized -with something external like Roam or Notion, so I think it'd be better to -build one of those integrations instead. - -## Stocks, News - -Even has a dashboard card which is a stock ticker. I think this is probably -misguided; while there do exist people who would want and benefit from a stock -tracker on their glasses, I do'm not sure there exist people who would want and -benefit from a stock tracker that isn't a professional-tier tool connected -to carefully chosen low-latency data sources, and neither I (nor Even) can -build that tool. - -The News dashboard card is more broadly useful, but has a similar issue: a -generic news ticker isn't really useful without a much higher degree of -customization. Probably the right thing to build instead is more like an RSS -reader. - -## Ring Connection Management - -The stock Even app is responsible for pairing the glasses with the ring (ie, -sending the glasses a configuration messages telling it to connect to the -ring's MAC address). If you pair them in the stock app and then switch to a -custom app, they remain paired indefinitely (this configuration is stored in -flash), and our custom app can make use of the fact that they're paired. - -We don't yet have support for connecting or disconnecting the glasses and ring -to/from each other. For latency reasons, it would be nice if we could unpair -the glasses from the ring and connect the phone to it directly; we don't yet -support that either. - -## Ring Health Metrics - -The R1 ring contains an oximeter-style LED sensor and an accelerometer, which -can be used for a step count, heart rate measurement, etc. I'm not sure -whether the sensor interpretation is done within the firmware (in which case -it would make sense to display them in the UI) or in the Even phone app (in -which case we'd be reimplementing all the interpretation algorithms, which -isn't worth it. - -## Navigate - -Even has a Navigate app, which is basically Google Maps for the specific use -case of "name a destination, get turn by turn directions to there". I believe -this is using Mapbox, which takes care of most of the heavy lifting (if you -have an API key). I don't think Even's version is very good (they ran into -trouble with image-upload performance preventing a good map display size), so -it's likely not too hard to do better. One caveat is the location permission; -Android normally only wants to prompt for location permission "while using the -app", but Android's idea of in use (phone screen unlocked and window focused) -doesn't match our idea of in use (G2 glasses on, phone could be locked). This -app also gets a lot better if we figure out how to use the device's -magnetometer (if it indeed has a working magnetometer). - -## Distance/Viewport Configuration - -During glasses onboarding, the stock firmware has you select a vertical offset -for the screen image. This accounts for glasses resting at different positions -on your face (the optics don't let you see the whole screen area), but cuts -the usable screen height from 480px to 288px. There is also a "distance" -config option in the stock Android app, which adjusts the horizontal position -of the image on the two lenses separately to create the impression of the -image being closer or farther; this cuts the usable screen width from 640px to -576px. Unlike the vertical size reduction, this 64px reduction is not -optically necessary. - -## Even AI -## Translate and Conversate - -## Other Configuration - -Brightness, head-up trigger - - -# Other Apps - diff --git a/notes/hermes-assistant.md b/notes/hermes-assistant.md deleted file mode 100644 index 5aab3885..00000000 --- a/notes/hermes-assistant.md +++ /dev/null @@ -1,41 +0,0 @@ -# Hermes agent as the assistant - -The "Hermes agent" assistant model sends what you say to a -[Hermes Agent](https://github.com/NousResearch/hermes-agent) gateway through its -OpenAI-compatible API server and streams the answer back to the glasses. Hermes -runs its own tools and keeps the conversation: each Faceclaw assistant -conversation maps to one Hermes session (`X-Hermes-Session-Id`), and only the -latest message is sent. Faceclaw's own glasses tools are not offered to Hermes. -The session ID is saved with the conversation, so a conversation restored after -an app restart continues the same Hermes session. - -A Hermes turn may run for up to 20 minutes (other models get 2), because Hermes -runs whole multi-tool tasks in one request and closing the stream interrupts the -run on the server. A run that fails or is interrupted on the server shows as an -error on the glasses, with any partial answer kept. - -## Enable the API server in Hermes - -In the Hermes config, enable the `api_server` platform and bind it where the -phone can reach it, then set a key in Hermes's `.env` and restart the gateway: - - hermes config set platforms.api_server.enabled true - hermes config set platforms.api_server.extra.host
- hermes config set platforms.api_server.extra.port 8642 - echo "API_SERVER_KEY=$(openssl rand -hex 24)" >> ~/.hermes/.env - -Check it with `curl -H "Authorization: Bearer " http://:8642/v1/models`. - -## Settings (Settings > Assistant) - -- **Assistant model**: Hermes agent. Disabled until the URL is set. -- **Hermes URL**: e.g. `http://host:8642/v1` (a bare host gets `/v1` added). -- **Hermes key**: the `API_SERVER_KEY` value. - -On iOS, App Transport Security blocks plain `http://` to IP addresses unless -`App_Resources/iOS/Info.plist` has an `NSExceptionDomains` entry for that host -(`NSExceptionAllowsInsecureHTTPLoads`); otherwise serve the API over https, for -example with `tailscale serve`. - -Actions that Hermes holds for approval (`approvals.mode manual`) cannot be -approved from the glasses; approve them from Hermes itself. diff --git a/notes/ios-protocol-sync.md b/notes/ios-protocol-sync.md deleted file mode 100644 index ec419a95..00000000 --- a/notes/ios-protocol-sync.md +++ /dev/null @@ -1,66 +0,0 @@ -# iOS protocol synchronization — 2026-09-17 - -Merged `main` at `1727474` into `ios-port`. The iOS session now uses the -same transport contract as Android's `CfwTransport`, `CfwMessageWindow`, -`OutboundMessage`, and `BleImageOptimizer`. - -- The EvenHub page contains only the input text container. Display commands - and cleanup use SID `0xf0`, without image-container or image-update headers. -- Mode 6 full frames and mode 3 bounding boxes contain uncompressed RLE. - Persistent zlib with SYNC_FLUSH runs in wire order at the transport layer. - Record lengths and checksums describe the same data as Android; packet - sizes respect CoreBluetooth's maximum write value length (ATT MTU minus 3). -- Commands larger than the uint16 record limit become independent 64-row - mode 3 bands. Frame metrics advance only when all bands complete. -- The three-command window requires ACKs for both processing lenses on the - ingress BLE link, with matching stream ID, ordinal, decoded size, and CRC. - Primary and redundant ACKs are validated before application; completion - remains ordered even when replies arrive out of order. -- NACKs or 500 ms ACK timeouts replay the entire unresolved window, using - fresh stream IDs and reset compression contexts. Three retries are allowed - before reconnecting and invalidating the delta base. Deadlines begin after - writes finish. Old queued attempts and late ACKs cannot complete replays. -- Firmware acceptance uses shared numeric revision checks (currently - Faceclaw/13 or newer), replacing the old EVENCFW feature-token check. - Legacy or stock firmware is rejected before display startup. - -Merge conflicts retained iOS voice restrictions while preserving main's -asynchronous transcription finalization. The iOS voice-state bridge supplies -the new listening/detail fields. Test harnesses account for platform globals -and the shared SVG rasterizer. - -Validation: TypeScript checking and signed physical-device iOS build pass. -The simulator build launches and renders the main screen and shared launcher. -The final signed app is installed on the development iPhone. The user renewed -developer-certificate trust after iOS initially blocked launch. A subsequent -NativeScript deployment launched successfully, connected to the glasses, and -received both-lens ACKs for a display frame. -The transport parity test compiles Android's production Java encoder and -compares complete packets, including persistent history, reset, MTU limits, -and raw fallback for incompressible maximum-size records. Session tests cover -ACK identity, two-lens completion, ordered pipelining, band reconstruction, -NACK replay, timeout/retry limits, microphone/input responsiveness, and cleanup. - -The full suite has 420 passing tests and 21 failures. All 21 reproduce on a -clean archive of main at 1727474: compass, pinball/touch, and terminal test -harness/behavior checks. There are no additional failing tests from this merge. - -Further hardware acceptance: scroll and animate the display, visually check -both lenses, exercise mic and ring input, lock the phone, and disconnect/reconnect. -The initial successful frame does not establish sustained radio reliability. - -## Development deployment and stale LiveSync code - -Installing the IPA with `devicectl` alone left an old September 8 JavaScript -bundle in `Library/Application Support/LiveSync/app` in the app's data container. -NativeScript debug startup used that bundle instead of the newly installed -one, causing the obsolete `img640, fbguard, wearnotify` firmware rejection. -Refreshing through NativeScript fixed the issue without clearing app settings: - -```sh -npm run run:ios -- --device 00008150-000C383921A3C01C --no-hmr --no-watch -``` - -Use this deployment path when updating an existing development installation. -The refreshed on-device `bundle.mjs` was copied back and verified against the -prepared bundle; it contains the shared revision-13 check and SID `0xf0`. diff --git a/notes/media-resume-via-media-key.md b/notes/media-resume-via-media-key.md deleted file mode 100644 index 0b052996..00000000 --- a/notes/media-resume-via-media-key.md +++ /dev/null @@ -1,13 +0,0 @@ -# Music: "Resume" action via media-key dispatch - -Follow-up to the MediaBrowserService browsing support. When no media session -is active, `AudioManager.dispatchMediaKeyEvent(KEYCODE_MEDIA_PLAY)` (down+up -pair) is routed by the system to the *last* media app's manifest-declared -`MEDIA_BUTTON` receiver, even if that app's process is dead — the same -mechanism a Bluetooth headset play button uses. Most players respond by -resuming the last queue. - -Idea: add a "Resume last" row to the Music app's no-active-session screen -(next to the browse entry). ~5 lines in `FaceclawMediaController` plus one UI -row; no per-app negotiation needed. Caveat: it's fire-and-forget — no feedback -if nothing handles it, so the UI should just wait for a session to appear. diff --git a/notes/mentraos-compat-assessment.md b/notes/mentraos-compat-assessment.md deleted file mode 100644 index 274d7d84..00000000 --- a/notes/mentraos-compat-assessment.md +++ /dev/null @@ -1,133 +0,0 @@ -# MentraOS app compatibility for Faceclaw — feasibility assessment (2026-07-23) - -MentraOS: https://github.com/Mentra-Community/MentraOS (MIT), docs at https://docs.mentraglass.com. - -## TL;DR - -Technically feasible with moderate effort, but the timing is awkward: MentraOS is -**deprecating its entire current app API**. MentraOS 3.0 ships **Aug 3, 2026** and -cloud-SDK apps stop working on it (a "MentraOS Legacy" mode preserves them through -**October 2026**). The replacement — a **Miniapp SDK** where apps are phone-local JS -bundles running in a WebView — gets a public spec in **September 2026**, and that model -is a *much* better fit for Faceclaw than the cloud model. Recommendation: wait for the -miniapp spec and target that; only build a minimal cloud-protocol emulator if access to -the existing app corpus is wanted in the interim. - -## MentraOS app model (current, "Cloud SDK" / TPA era) - -- Apps are developer-hosted server processes (Node/Bun, `@mentra/sdk`). Chain: - glasses → phone (BLE) → MentraOS Cloud (WS) → app server (WS). -- Start flow: cloud POSTs a `session_request` webhook `{sessionId, userId, websocketUrl}` - to the app's registered URL; the app dials back to `websocketUrl` and sends - `tpa_connection_init {packageName, sessionId, apiKey}`; cloud acks with settings + - device capabilities. **The webhook carries the WS URL, so an unmodified `@mentra/sdk` - app can be pointed at any server** — that is the compat hook. -- Protocol: JSON over WebSocket (binary frames for audio). 28 app→cloud + 28 cloud→app - message types, defined in `cloud/packages/sdk/src/types/` (`message-types.ts`, - `streams.ts`, `layouts.ts`, `webhooks.ts`, `capabilities.ts`). The TS types are the spec. -- Subscriptions: `subscription_update` with a full list of stream types, some - parameterized (`transcription:en-US`, `touch_event:forward_swipe`). Events arrive in a - `data_stream` envelope. -- Display: 7 layout types — `text_wall`, `double_text_wall`, `dashboard_card`, - `reference_card`, `bitmap_view` (base64), `bitmap_animation`, `clear_view` — sent as - `display_event {view: 'main'|'dashboard', layout, durationMs?, forceDisplay?}`. - Cloud throttles displays to 1 per 300 ms. Dashboard = shared look-up surface. -- Streams: `transcription` (`{text, isFinal, ...}`), `translation`, `VAD`, `audio_chunk` - (PCM), `button_press`, `touch_event`, `head_position` (up/down), `glasses_battery_update`, - `phone_battery_update`, `location_update`, `phone_notification`(+dismissed), - `calendar_event`, plus camera/RTMP/LED families irrelevant to G2. -- Other: ElevenLabs TTS (`audio.speak`), per-app settings (schema in dev console, 9 - setting types, live `settings_update`), webview UI with signed-token auth, cloud KV - storage, app-to-app messaging. -- Hardware abstraction: `Capabilities` flags per device. **Even Realities G2 is a - first-class device** (`capabilities/even-realities-g2.ts`): display 640×200 (their - rendering canvas, not the 576×288 EvenHub raster), maxTextLines 5, bitmap-capable, - capacitive swipe bar with TAP/DOUBLE_TAP/TRIPLE_TAP/PRESS_HOLD/SWIPE_UP/SWIPE_DOWN, - IMU, mic, no camera/speaker/LED. -- Licensing: MIT throughout; cloud is self-hostable in principle (Bun + MongoDB + S3 + - third-party STT/TTS keys) but heavy. - -## Coming model (Miniapp SDK, MentraOS 3.0) - -Miniapps are JS bundles (`index.html` + `miniapp.json`) running **on the phone** inside -the MentraOS host app's WebView ("Mentra Runtime"), packages `@mentra/miniapp` + -`@mentra/miniapp-cli`, sideloaded via QR. API is analogous but smaller: -`session.transcription`, `session.display.showTextWall`, `session.storage`. Example: -https://github.com/Mentra-Community/LiveCaptionsLocalSdkMiniapp. Public spec Sept 2026. - -## Fit with Faceclaw - -Faceclaw's worker-per-app architecture is unusually favorable: apps are already isolated -contexts speaking a small typed JSON vocabulary (`WorkerAppMessage`/`WorkerAppReply` in -`app/ui/shell/worker-window.ts`), with pixels out-of-band. The compat layer would be one -compiled-in "MentraOS host" app (a peer of `WorkerAppHost`) that presents each external -app as a `ShellWindow` and: - -- **Rasterizes layouts** → trivial. All 7 layout types map onto `GrayImage` + - `drawTextWrapped`/`bitBlt` into the 540×260 `APP_VIEWPORT`. Bitmaps need scaling from - their 640×200/576×136 assumptions; monochrome already matches. -- **Maps input** → good. `DashboardInputEvent` (`click`/`double-click`/`long-press`/ - `scroll-up`/`scroll-down`) covers their G2 button-event list almost 1:1 (TRIPLE_TAP - missing). Work item: input is currently routed only to the focused window; MentraOS - wants multi-subscriber fan-out (background apps), so a broadcast layer is needed. -- **Transcription** → the load-bearing stream, and Faceclaw already has it: - `voiceControlBridge.onTranscript` (`{text, isFinal}`, replace semantics — matches - MentraOS's `TranscriptionData` model well). Onboard Moonshine is English-only; - parameterized languages would fall back to ElevenLabs/Whisper cloud providers. - Needs multi-consumer fan-out and a policy for when the mic is open (MentraOS keeps it - effectively continuous; Faceclaw is PTT/wakeword-oriented — battery implications). -- **VAD** → sherpa endpointing exists. **audio_chunk** → PCM already surfaced on the - ElevenLabs path (`onPcm`). -- **Battery / phone notifications / calendar** → data already flows in Faceclaw - (top-bar battery, notification listener, calendar app); just needs re-emission as - streams. -- **head_position** → gap. Faceclaw exposes raw IMU only (no gesture classification); - the look-up dashboard gesture would need a small accelerometer classifier (cmd 19 - stream exists, see imu-accelerometer memory). -- **TTS / audio out** → G2 has no speaker; MentraOS plays through the phone. Faceclaw - would need phone-side audio playback (new but simple Android work). -- **Settings** → MentraOS per-app settings map onto the shared `FaceclawSettings` store - with a `mentra..*` prefix; the 9 setting types map onto - `ConfigSetting` subclasses, mostly. -- **Not applicable on G2**: camera/photo/RTMP/LiveKit, LED, WiFi setup — can be - rejected via capabilities, which the protocol explicitly supports. - -## The hard parts - -1. **Who plays "cloud"?** Cloud-SDK apps dial out to a WS URL given in a webhook, so - Faceclaw must (a) run a WS **server** — none exists anywhere in the codebase today, - all networking is outbound okhttp client (`FaceclawWebSocket`, g2mirror pattern) — - reachable from wherever the app server runs (fine on LAN; NAT pain otherwise), and - (b) send the session webhook itself (simple outbound HTTP POST). A phone-hosted - mini-cloud with Java-WebSocket/NanoWSD listening on LAN is the sane MVP; self-hosting - their actual cloud (Bun+Mongo+Docker) is overkill. -2. **Session lifecycle**: connection init/ack, `subscription_update` bookkeeping, - `reconnect`/`reconnect_ack/rejected/deferred`, `app_stopped`, per-app permission - gating. This is bookkeeping-heavy but not deep. -3. **Display arbitration**: foreground vs `background`+`forceDisplay` apps, `durationMs` - auto-revert, the shared dashboard view. Faceclaw's shell has focus/foreground but no - equivalent of background-app display preemption or a look-up dashboard. -4. **App discovery/registration UX**: no dev console; needs a simple "add app by URL + - API key" settings flow (API key check can be a no-op or shared secret). - -## Effort estimate - -MVP covering the bulk of display-glasses apps (live captions, teleprompters, dashboards, -notification relays): ~10–15 message types + ~8 stream types + webhook sender + LAN WS -server + layout rasterizer + settings mapping. Roughly a few weeks of focused work, with -transcription-stream policy and the WS server being the riskiest pieces. Full-surface -emulation (webview auth, app messaging rooms, TTS, location tiers) is substantially more -and mostly not worth it. - -## Recommendation - -- **Don't invest in the cloud-TPA protocol as the long-term target** — the vendor is - killing it within ~3 months of now, and the ecosystem's apps will migrate to miniapps. -- **Target the Miniapp SDK when its spec lands (Sept 2026).** It's strictly easier for - Faceclaw: JS bundle in an Android WebView with an injected `session` bridge — no - server, no webhooks, no reconnect protocol — and Faceclaw already has every underlying - facility the bridge needs. The rasterizer/input/transcription plumbing built for it - would be shared with any interim TPA emulator anyway. -- If interim access to existing cloud-SDK apps matters, build the minimal LAN emulator - described above, scoped to text layouts + transcription + button streams, and treat it - as disposable. diff --git a/notes/voice-assistant-design.md b/notes/voice-assistant-design.md deleted file mode 100644 index 675d1c8d..00000000 --- a/notes/voice-assistant-design.md +++ /dev/null @@ -1,466 +0,0 @@ -# Voice Assistant Architecture - -Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability -ontology, misc fixes); revised 2026-08-01 (external bridge landed). -**Phases 1 and 2 are implemented**, and **phase 3 + the OpenClaw part of -phase 4 landed 2026-08-01** with one architecture change: the bridge server -is not a standalone package — it *is* the OpenClaw plugin -(`~/repositories/faceclaw-agent-bridge`, installed on the OpenClaw host). -The plugin hosts the websocket endpoint the phone dials into, runs -utterances as embedded OpenClaw agent turns (`runEmbeddedAgent` with a -dedicated session, default key `faceclaw:glasses`), and exposes the -glasses' MCP-served tools to the agent via two fixed meta-tools -(`glasses_list_tools` / `glasses_call` — fixed names because OpenClaw's -manifest `contracts.tools` wants static declarations, while the glasses -toolset is dynamic). The generic "MCP face for any agent" from the original -design is unbuilt; the wire protocol is unchanged, so a standalone bridge -for other agents can be added later without phone changes. - -## Goal - -A default voice assistant reachable from anywhere in the shell: hold the -button, speak, and the utterance goes to an agent that can answer in text on -the lenses and act on the glasses through tools. The agent is pluggable: - -- **Direct mode**: the phone runs the agent loop itself against the Anthropic - Messages API (tool-use loop, conversation state on-phone). -- **External mode**: the utterance is routed to the user's own long-running - agent (OpenClaw, Hermes, or anything similar running on their machine), and - that agent gets access to the glasses' tools while it works. - -Tools come from two sources, with four availability tiers (see "Tool -availability ontology" below): - -- **Glasses/system tools**: provided by the shell, always available (show - text, notifications, screen state, open app, ...). -- **App tools**: contributed by apps. Each app tool declares how much of the - app needs to exist for the tool to be live: merely **installed** ("start a - 5 minute timer" must not require the Timer app to be open), **open** - ("next track" needs a running Music window, foreground or not), or - **foreground** ("scroll down" only makes sense aimed at the foreground app). - -## What we already have - -- **PTT capture UI**: `VoiceInputLayer` (app/ui/apps/voice-input.ts) handles - push-to-talk, live transcripts, and a Send/Continue/Discard menu; `Shell` - opens it on long-press and delivers Send to the foreground window's - `receiveTextInput` (app/ui/shell/shell.ts). -- **Wakeword**: On-device classifier listens for the wakeword "Hey Even", - which sends a message to the phone which wakes the screen and starts voice - input. -- **STT**: `voiceControlBridge` with onboard Moonshine, Whisper, or ElevenLabs - cloud. -- **LLM client**: `streamAnthropicMessage` (app/native/anthropic.ts) — raw - SSE streaming over `FaceclawSseRequest`. Text-only today; no tool use. -- **Worker app protocol**: `WorkerAppMessage`/`WorkerAppReply` - (app/ui/shell/worker-window.ts) — small-JSON postMessage protocol between - shell and app workers; the natural place to add tool declarations. -- **Dial-out websocket precedent**: `G2MirrorClient` — JSON over a websocket - to a server on the user's machine (tailscale host/port/token settings). - The external-agent bridge should copy this shape. - -## Architecture overview - -``` - main isolate (shell) - ┌───────────────────────────────────────────────────────────┐ - │ Shell ──(PTT)──> VoiceInputLayer ──> AssistantSession │ - │ │ │ - │ AssistantLayer (overlay UI) <──streaming──┤ │ - │ │ │ - │ AssistantBackend │ - │ ┌────────┴─────────┐ │ - │ │ Direct │ External│ │ - │ └────┬────┴────┬────┘ │ - │ │ │ │ - │ ToolRegistry <────────────────────────┴─────────┘ │ - │ ├─ system tools (in-process handlers) │ - │ ├─ app "installed" tools (shell-side or launch-on-call) │ - │ └─ app "open"/"foreground" tools (proxied to app worker) │ - └──────────┬──────────────────────────────────────┬─────────┘ - │ postMessage │ - app worker (e.g. terminal) Anthropic API (SSE) - set-tools / tool-call / or agent bridge (WS) - tool-result on the user's machine -``` - -Everything assistant-side lives on the main isolate. Rationale: tool handlers -need shell state (window list, focus, notifications, settings), the agent -loop is I/O-bound not CPU-bound, and the overlay UI is a shell layer anyway. -If conversation rendering ever gets expensive it can move to a worker later -without changing the tool protocol, which is already async message passing. - -### Core pieces - -1. **`ToolRegistry`** (new, `app/assistant/tool-registry.ts`) - - Holds `ToolSpec`s and handlers; answers `listTools(scopeFilter)` and - `callTool(name, args) -> Promise`. - - Emits `onToolsChanged` (needed for MCP `tools/list_changed` in external - mode, and consulted at each turn boundary in direct mode). - -2. **`AssistantBackend`** (interface) with two implementations: - - `DirectAnthropicBackend`: tool-use loop over the Messages API. - - `ExternalAgentBackend`: websocket bridge to the user's agent. - -3. **`AssistantSession`**: one conversation. Owns history/turn state, - exposes `sendUtterance(text, ctx)`, `cancel()`, and streaming callbacks - (`onTextDelta`, `onToolActivity`, `onTurnDone`, `onError`). Both backends - implement the same session surface so the UI is backend-agnostic. - -4. **`AssistantLayer`**: shell overlay showing streamed reply text, a status - line for tool activity ("→ calendar.list_events"), and a small menu - (Follow-up / Done). Double-click cancels the in-flight turn. - -## Entry points and routing - -- **Wakeword** Saying "Hey Even" wakes the screen if necessary and opens the - dictation dialog. For one-off voice commands, this will be the main entry - point. -- **Context menu** The sidebar context menu and app context menu already have - a "Voice input" option, which opens the dictation dialog whose Send goes to - the foreground window. We'll split the Send option in this dialog into two - options, "Send to Assistant" and "Type Into App". If the entry point was via - context menu, "Type Into App" is highlighted by default; if the entry point - was the wakeword, "Send to Assistant" is highlighted by default instead. - (The capture UI is the existing `VoiceInputLayer` flow, with Send handing off - to `AssistantSession` and morphing into `AssistantLayer`). - -## Tool model - -```ts -type ToolAvailability = - | "always" // system tools; no app involved - | "installed" // app tool, live whenever the app is installed — - // the app need not have any window open - | "open" // app tool, live while the app has a window open, - // foreground or backgrounded - | "foreground"; // app tool, live only while the app owns the - // foreground window - -type ToolSpec = { - name: string; // namespaced: "glasses.show_notification", - // "app.terminal.send_input" - description: string; - inputSchema: object; // JSON Schema - availability: ToolAvailability; - /** May be invoked by the external agent outside an active voice turn. */ - proactive?: boolean; - timeoutMs?: number; // default 10s; registry enforces -}; - -type ToolResult = - | { ok: true; content: string } // text payload back to the model - | { ok: false; error: string }; -``` - -### Tool availability ontology - -Every tool sits in exactly one tier, and the tier answers two questions: *when -does the tool appear in `listTools()`*, and *who handles the call*. - -| Tier | Live when | Handler | Examples | -|---|---|---|---| -| `always` | always | shell, in-process | `glasses.get_state`, `notifications.list` | -| `installed` | app installed (launcher entry exists) | shell-side handler registered with the app entry, or launch-the-app-then-proxy | `timer.set` ("start a 5 minute timer" must work with no Timer window), `music.play` (cold-start playback) | -| `open` | app has ≥1 open window, foreground or not | app worker (or in-process window) via tool-call proxy | `music.next_track`, `terminal.send_input` | -| `foreground` | app owns the foreground window | app worker via tool-call proxy | `teleprompt.scroll_down`, anything meaning "act on what I'm looking at" | - -Mechanics per tier: - -- **`installed`** tools cannot be declared by the worker (it may not be - running), so they are declared statically alongside the app's launcher - registration. Two handler flavors: (a) a shell-side handler function - registered with the entry — right for tools that don't actually need the - app UI (timers, starting playback); (b) *launch-on-call*: the shell opens - the app (as if launched from the launcher), waits for its `set-tools` - declaration, then proxies the call. Start with (a) only; (b) adds a - launch-latency/timeout dance that no phase-1/2 tool needs. -- **`open`** and **`foreground`** tools are declared by the running app via - `set-tools` (worker apps) or direct registration (in-process windows). - The registry filters `foreground` tools by whether the declaring app owns - the current foreground window at list time and again at call time — - a `foreground` tool called after a foreground swap fails with a normal tool - error rather than acting on a window the user is no longer looking at. -- A tier is a *floor*, not a routing rule: an `installed` tool stays listed - while the app is open or foreground. If an app wants different behavior when - its window exists (e.g. `timer.set` also shows a countdown in the open - Timer window), the shell-side handler checks for the window itself. - -> **Flag:** with multiple windows per app (the worker protocol is -> per-window), tool names are namespaced per *app*, so two windows of one -> app declaring the same tool name would collide. Rule: `set-tools` -> replaces the app's toolset per window, and a call routes to the declaring -> window; if two windows declare the same name, last declaration wins and -> earlier ones are dropped with a logged warning. Revisit only if a real -> app needs per-window tools with the same name (none of the planned ones -> do). - -### System tools (initial set) - -| Tool | Notes | -|---|---| -| `glasses.show_alert` | short text popup on the lenses; proactive-capable | -| `glasses.list_windows` / `glasses.focus_window` / `glasses.open_app` | shell control | -| `glasses.get_state` | screen on/off, foreground app, battery levels, time | -| `glasses.read_screen` | text/description of the current foreground surface (start with window title + app-reported summary; pixel OCR is out of scope) | -| `calendar.list_events` | wraps FaceclawCalendarProvider | -| `media.play_pause` / `media.next` / `media.now_playing` | wraps FaceclawMediaController | -| `notifications.list` / `notifications.dismiss` | wraps notification listener | -| `timer.set` / `timer.cancel` | new, but trivial and high-value for an assistant | - -Keep the first release to ~8–12 tools; every tool costs prompt tokens and -model attention in direct mode. - -Note that `calendar.*`, `media.*`, and `timer.*` are conceptually -`installed`-tier app tools that happen to be implemented by in-process shell -code; listing them here just means the shell registers their handlers -directly. If any of them later becomes a real app, its tools move to the -app's launcher registration without the model-visible surface changing. - -### App tools - -Running apps declare `open`/`foreground` tools per window over the existing -worker protocol (`installed` tools are declared statically with the launcher -entry, per the ontology above): - -```ts -// WorkerAppReply additions (worker -> shell) -| { type: "set-tools"; windowId: string; tools: ToolSpec[] } // replaces the set -| { type: "tool-result"; callId: string; result: ToolResult } - -// WorkerAppMessage additions (shell -> worker) -| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown } -``` - -Registry rules: - -- Liveness follows the availability tier: `open` tools are removed when the - app's last window closes, `foreground` tools additionally drop out of - `listTools()` when the app loses the foreground. Either transition fires - `onToolsChanged`; in-flight calls to a closed window resolve as errors, - and calls to a `foreground` tool are re-checked at dispatch time. -- Names are auto-prefixed `app..` by the registry so apps can't - shadow system tools or each other. -- The shell enforces `timeoutMs` on the postMessage round-trip; a hung - worker yields a tool error, not a hung turn (same philosophy as input - handling — the shell must survive a stuck app). -- In-process shell apps (calendar, music, ...) register handlers directly - with the registry; the worker protocol is just the remote flavor of the - same registration. -- **Mid-turn availability changes** (foreground swap, window close): direct - mode re-lists tools at each loop iteration (tool set is per-API-call - anyway); external mode sends MCP `notifications/tools/list_changed`. A - call to a tool that just vanished returns a normal tool error the model - can react to. - -First app to wire up: the terminal — -`app.terminal.send_input` (type into the attached session; `open`, -proactive: false — "rerun the build" should work while the terminal is -backgrounded), `app.terminal.read_screen` (current grid contents; `open`), -`app.terminal.list_sessions` (`open`). This immediately enables "tell the -terminal to rerun the build" style commands and doubles as the reference -implementation — deliberately all `open`-tier; the first `foreground` tool -should come from an app where focus genuinely matters (teleprompt scrolling). - -## Direct mode: agent loop on the phone - -Extend `app/native/anthropic.ts`: - -1. **Streaming tool-use support**: handle `content_block_start` with - `tool_use`, accumulate `input_json_delta`, surface complete tool_use - blocks alongside text deltas. (Today the client only reads `text_delta`.) -2. **`runAgentTurn` loop** (new, `app/assistant/direct-backend.ts`): - messages + `tools` from the registry → stream → if `stop_reason == - "tool_use"`, execute each call via the registry, append `tool_result` - blocks, continue; until `end_turn`/`refusal` or a safety cap - (max 8 loop iterations, max ~2 min per turn). -3. **Session state**: history kept in memory per `AssistantSession`, - trimmed from the head past a token budget. Sessions idle-expire - (~10 min) so a new PTT starts fresh; "Follow-up" continues the session. -4. **System prompt**: glasses context — output renders on a 576×288 - monochrome HUD, so answer in 1–3 short sentences, no markdown, no lists - unless asked; prefer acting via tools over describing; current time, - foreground app, and screen state are injected per turn. -5. Model `claude-sonnet-5`, `effort: "low"` (same latency posture as the - dictation refiner). Model/effort become settings later if needed. - -## External mode: bridge to the user's agent - -### Topology - -The phone **dials out** (g2mirror pattern — phones don't accept inbound -connections reliably; tailscale host + port + token settings) to a small -**bridge server** co-located with the user's agent. One websocket, JSON -frames, three multiplexed channels: - -``` -{ v: 1, chan: "ctl" | "chat" | "mcp", ... } - -ctl: hello {deviceName, token, capabilities}, hello-ack, ping/pong, error -chat: utterance {turnId, text, ctx} (phone -> agent) - text-delta {turnId, text} (agent -> phone) - tool-activity {turnId, label} (agent -> phone, optional status) - turn-done {turnId, stopReason} / turn-error {turnId, message} - cancel {turnId} (phone -> agent) -mcp: { msg: } (bidirectional) -``` - -On the `mcp` channel the **phone is an MCP server** (MCP roles come from the -initialize handshake, not from who dialed). It serves `tools/list` from the -ToolRegistry, `tools/call` into it, and emits `tools/list_changed` on -foreground changes. This means the agent-side integration needs zero -Faceclaw-specific tool code — the glasses appear as a normal MCP server. - -### Agent-side adapters - -The bridge server is a small standalone package (`faceclaw-agent-bridge`, -sibling of g2mirror) with two faces: - -1. **MCP face**: re-exposes the phone's MCP channel as a localhost - streamable-HTTP MCP endpoint. The user registers - `http://localhost:/mcp` with their agent (OpenClaw, Hermes, Claude - Code, anything MCP-capable) exactly like any other MCP server. When the - phone is disconnected, `tools/list` returns empty rather than erroring. -2. **Chat face**: pluggable per-agent adapter that forwards `utterance` - frames into the agent and streams replies back: - - **OpenClaw**: a plugin that injects the utterance as an inbound message - on a dedicated "glasses" channel/session and relays the streamed reply. - - **Hermes**: equivalent module against its session API. - - **Fallback adapter**: spawn a configured CLI (e.g. `claude -p --resume`) - per turn — lowest-fidelity but makes the bridge useful with no plugin. - -Utterance `ctx` carries `{foregroundApp, screenOn, localTime}` so the remote -agent has the same situational grounding the direct loop gets in its system -prompt. - -### Proactive agent actions - -Because the MCP channel is live whenever the bridge is connected, the -external agent can call glasses tools **outside a voice turn** — e.g. push a -notification to the lenses when a long job finishes. Gating: - -- Only tools marked `proactive: true` are callable outside an active turn - (registry-enforced; enforced on the phone, never trusted to the bridge). - Availability tiers apply to proactive calls unchanged: a proactive - `installed` tool (e.g. `timer.set`) works anytime, while a proactive call - to an `open`/`foreground` tool whose app isn't in the right state gets the - same tool error a mid-turn call would. -- Master setting `assistant.allowProactive` (default on — it's a marquee - feature — but visible and easy to turn off). -- Proactive display actions are rate-limited (e.g. 6/min) and never turn - the screen on unless the user enabled that specifically. - -## Settings - -New "Assistant" section in dashboard settings: - -- `assistant.backend`: `anthropic` | `external` (default `anthropic` if an - API key is set, else `external` if a bridge host is set). -- `assistant.bridgeHost` / `assistant.bridgePort` / `assistant.bridgeToken` - (mirrors the terminal.* trio). -- `assistant.allowProactive`: boolean. -- `voice.skipConfirmationAfterWakeword`: Wakeword-triggered input sends to the - assistant immediately at the end of an utterance, rather than waiting for a - menu selection. -- Reuses existing `llm.anthropicApiKey` for direct mode. - -## Security notes - -- Bridge auth: bearer token in the `hello` frame, TLS optional because the - expected transport is tailscale (same stance as g2mirror); document that - plainly. -- The phone enforces all tool gating (proactive flags, rate limits, - timeouts). The bridge and agent are treated as honest-but-fallible. -- Tool results may contain private data (calendar, notifications); external - mode ships them to the user's own machine only — no third-party hop. In - direct mode they go to the Anthropic API like any other request content. -- App tools execute inside the app's worker with the app's existing - privileges; the registry adds no new capability beyond what the app could - already do. - -## Implementation plan - -**Phase 1 — direct assistant, system tools only.** ✅ Landed 2026-07-24 -(builds + typechecks; not yet hardware-tested). -ToolRegistry (`app/assistant/tool-registry.ts`) with the full availability-tier -model but only `always` tools wired; 8 system tools (`glasses.get_state`, -`glasses.show_alert`, `calendar.list_events`, `media.now_playing`/`play_pause`/ -`next`, `notifications.list`/`dismiss`) in `app/assistant/system-tools.ts`; -tool-use streaming in `anthropic.ts` (content-block assembly, `tools` param); -`DirectAnthropicBackend` + `AssistantSession`; `AssistantLayer` overlay -(`app/ui/apps/assistant.ts`); `VoiceInputLayer` reworked to route via a -`sendTargets` list ("Send to Assistant" / "Type Into App") with entry-point -default highlight and wakeword skip-confirmation auto-send; -`shell.showAlert` popup; `assistant.skipConfirmationAfterWakeword` setting. -*Deliverable: speak wakeword, ask "what's on my calendar", get an answer -on-lens.* - -**Phase 2 — app tools.** ✅ Landed 2026-07-25 (builds + typechecks; not yet -hardware-tested). -Worker protocol extended with `set-tools` / `tool-result` (WorkerAppReply) and -`tool-call` (WorkerAppMessage). `ToolRegistry.setAppTools`/`removeAppTools` -key tools by windowId, prefix names `app..`, and gate `foreground` -tools on an `isForeground()` predicate (checked at list and dispatch time). -`WorkerAppHost` proxies calls to its worker (callId round-trip, 15s host -timeout backstop on top of the registry's per-tool timeout), registers a -window's tools on `set-tools`, and withdraws them + fails in-flight calls on -window close. Direct mode now re-lists tools each loop iteration. -Terminal is the reference: the hub window declares `send_input` / -`read_screen` / `list_sessions` (all `open`-tier); send_input/read_screen act -on the active view (foregrounded → last-active → sole view). *Deliverable: -"run the build again" typed into the terminal even while it's backgrounded.* - -*Not yet done from the Phase 2 design:* `installed`-tier registration on -launcher entries (no `installed` app tool exists yet — timer stays deferred); -in-process-window tool registration (only worker apps wired); the per-window -same-name collision handling (the terminal sidesteps it by declaring all tools -on the single hub window). - -**Phase 3 — external bridge.** ✅ Landed 2026-08-01 (see the status note at -the top for the architecture change: bridge server == OpenClaw plugin). -Phone side: `app/assistant/bridge-client.ts` (`assistantBridge` singleton — -dial-out FaceclawWebSocket, hello/token auth, auto-reconnect with 1s→60s -backoff, started at boot by the dashboard controller when configured), -`app/assistant/mcp-server.ts` (MCP over the `mcp` channel: initialize / -tools/list / tools/call from the ToolRegistry, list_changed on -onToolsChanged, proactive gating + 6/min rate limit enforced on-phone), -external mode in `AssistantSession` (`AssistantBackendConfig` union selects -direct vs external; external history lives on the agent's machine), settings -(`assistant.backend`, `assistant.bridgeHost/Port/Token`, -`assistant.allowProactive`) in a new Assistant settings section. -Server side: `~/repositories/faceclaw-agent-bridge` (own repo/package). -*Verified against the live OpenClaw on serac with a simulated phone -(`test/fake-phone.js`): auth, streamed turns, and both tool directions -work; a real model turn needs the OpenClaw instance to have provider auth.* - -**Phase 4 — OpenClaw/Hermes adapters + proactive polish.** Partially landed -2026-08-01: the OpenClaw adapter is the plugin itself (no separate chat -adapter needed); proactive gating + rate limits and `tools/list_changed` -are done. Remaining: Hermes (or generic-agent) support — likely a -standalone bridge speaking the same wire protocol with an MCP face + -CLI chat adapter; session persistence questions across reconnects -(currently: OpenClaw session is persistent, phone session is UI-only). - -**Hardware-test TODO (external mode):** end-to-end on the real phone + -glasses against serac — connect status, a streamed turn, cancel, follow-up, -proactive `glasses.show_alert` from another OpenClaw channel, and behavior -across phone network transitions (the reconnect backoff is untested on -device). - -## Open questions (deferred, not blockers): - -- Continuous conversation: after a turn ends, listen for a follow-up - utterance without requiring the wakeword/PTT again? There are some UI - design questions here; maybe multi-turn conversations take the form of an - Assistant app. -- Launch-on-call for `installed`-tier tools that genuinely need the app UI - (open the app, await `set-tools`, proxy the call) — deferred until a tool - needs it. -- `glasses.read_screen` fidelity beyond app-reported summaries? We probably - don't want to do OCR, but we might do something like extending the draw-text - draw call with an on-by-default option that also appends the drawn text to - a string that gets submitted along with the frame. -- Whether assistant conversations should be reviewable in the phone UI - (transcript log) — probably yes, cheap once sessions are objects. The - natural UI home for this would be inside an Assistant app window. - diff --git a/notes/whisper-server.md b/notes/whisper-server.md deleted file mode 100644 index a17357c8..00000000 --- a/notes/whisper-server.md +++ /dev/null @@ -1,53 +0,0 @@ -# Self-hosted Whisper transcription - -The "Whisper server (self-hosted)" transcription provider sends voice audio to -any OpenAI-compatible Whisper server instead of a cloud service: vLLM, speaches -(faster-whisper-server), LocalAI and similar. It uses the batch endpoint -`POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk -utterances are one request each; in Transcribe, each span ended by a pause is -one request. While a span is open its audio so far is re-sent every 1.5 s, so -text appears while you are still talking. A span that runs 25 s without a pause -(nonstop speech, music) is cut at its quietest moment in the last 5 s, so no -request outgrows Whisper's 30 s window; the pieces still arrive as one final. -On Android the WAV goes out through `FaceclawBinaryPost` (okhttp), since -NativeScript's http module copies a binary body into Java a byte at a time. - -## Server - -Any server exposing `/v1/models` and `/v1/audio/transcriptions` works, e.g. - - vllm serve openai/whisper-large-v3 --port 8004 - -## Settings (Settings > Voice) - -- **Transcription Provider**: Whisper server (self-hosted). Disabled until the - URL is set. -- **Whisper server URL**: the API root, e.g. `http://host:8004/v1`. A bare - `http://host:8004` gets `/v1` added. -- **Whisper server model**: empty uses the first model the server lists. -- **Whisper language**: an ISO code such as `en`, or empty to auto-detect. - A wrong fixed language makes Whisper translate rather than transcribe. -- **Whisper server key**: optional bearer token. - -## iOS: plain http - -iOS App Transport Security blocks plain `http://` to IP addresses (including -Tailscale's 100.x addresses) unless the app's Info.plist has an exception for -that host. Either serve the API over https (for example with `tailscale serve`) -or add your server to `NSExceptionDomains` in `App_Resources/iOS/Info.plist` -before building: - - 192.0.2.10 - NSExceptionAllowsInsecureHTTPLoads - -## Transcribe on iOS - -Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer -only handles short utterances). Its capture follows the glasses: a disconnect, -a glasses lock or backgrounding (for phone-mic capture) transcribes what was -already heard, and listening resumes once the glasses reconnect, unlock or the -app returns to the foreground. Without a glasses session the phone mic stands in -while Faceclaw is in the foreground. With "Save Transcribe audio" on (Voice -settings), each session's audio is kept as `transcribe-