forked from Blendman974/faceclaw
Make iOS app main screen match the Android UI
This commit is contained in:
1 parent
9f53847bb7
commit
6c3144cc31
30 files changed
+1013
-439
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_tab_keyboard@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 590 B |
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_tab_mic@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
@@ -20,8 +20,10 @@
|
||||
<true/>
|
||||
<key>NSBluetoothAlwaysUsageDescription</key>
|
||||
<string>Faceclaw uses Bluetooth to connect to your glasses and ring, display your apps, and receive gestures.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Faceclaw uses your phone microphone for voice input when your glasses are disconnected.</string>
|
||||
<key>NSSpeechRecognitionUsageDescription</key>
|
||||
<string>Faceclaw turns audio from your glasses microphone into text for the app you choose. Dictation uses on-device recognition.</string>
|
||||
<string>Faceclaw turns audio from your glasses or phone microphone into text for the app you choose. Dictation uses on-device recognition.</string>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>Faceclaw uses the camera to scan QR codes for app URLs you open in Developer.</string>
|
||||
<key>NSCalendarsUsageDescription</key>
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
/** On-device speech recognition fed ONLY by decoded G2 BLE microphone packets. */
|
||||
/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */
|
||||
@interface FaceclawSpeech : NSObject
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
+ (NSInteger)authorizationStatus;
|
||||
+ (NSInteger)microphoneAuthorizationStatus;
|
||||
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion;
|
||||
+ (void)requestAuthorization:(void (^)(NSInteger status))completion;
|
||||
/** Returns an actionable error, or an empty string when capture starts. */
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing;
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing;
|
||||
- (void)acceptPacket:(NSData *)packet;
|
||||
- (void)finish;
|
||||
- (void)cancel;
|
||||
|
||||
@@ -14,7 +14,11 @@
|
||||
@property(nonatomic, strong) dispatch_queue_t audioQueue;
|
||||
@property(atomic) NSUInteger generation;
|
||||
@property(nonatomic) NSUInteger pendingPackets;
|
||||
@property(nonatomic) BOOL accepting;
|
||||
@property(atomic) BOOL accepting;
|
||||
@property(nonatomic, strong) AVAudioEngine *phoneEngine;
|
||||
@property(nonatomic) BOOL phoneTapInstalled;
|
||||
@property(nonatomic) BOOL phoneSessionActive;
|
||||
@property(nonatomic, strong) id interruptionObserver;
|
||||
@property(nonatomic, copy) NSString *bestText;
|
||||
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
|
||||
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
|
||||
@@ -24,6 +28,12 @@
|
||||
if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL);
|
||||
return self;
|
||||
}
|
||||
+ (NSInteger)microphoneAuthorizationStatus { return [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]; }
|
||||
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger))completion {
|
||||
[AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ completion(granted ? 3 : 2); });
|
||||
}];
|
||||
}
|
||||
+ (NSInteger)authorizationStatus { return SFSpeechRecognizer.authorizationStatus; }
|
||||
+ (void)requestAuthorization:(void (^)(NSInteger))completion {
|
||||
[SFSpeechRecognizer requestAuthorization:^(SFSpeechRecognizerAuthorizationStatus status) {
|
||||
@@ -36,7 +46,15 @@
|
||||
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:NO];
|
||||
}
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:YES];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
[self cancel];
|
||||
if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized)
|
||||
return @"Allow Microphone access for Faceclaw in iPhone Settings.";
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US";
|
||||
@@ -44,8 +62,10 @@
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
self.decoder = [FaceclawLc3Decoder new];
|
||||
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
|
||||
if (!phone) {
|
||||
self.decoder = [FaceclawLc3Decoder new];
|
||||
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
|
||||
}
|
||||
self.recognizer.queue = NSOperationQueue.mainQueue;
|
||||
self.request = [SFSpeechAudioBufferRecognitionRequest new];
|
||||
self.request.requiresOnDeviceRecognition = YES;
|
||||
@@ -77,7 +97,11 @@
|
||||
}
|
||||
});
|
||||
}];
|
||||
[self emit:@{@"kind":@"status", @"message":[NSString stringWithFormat:@"Listening on glasses (%@, on-device)…", language]}];
|
||||
if (phone) {
|
||||
NSString *error = [self startPhoneAudio];
|
||||
if (error.length) { [self cancel]; return error; }
|
||||
}
|
||||
[self emit:@{@"kind":@"status", @"message":[NSString stringWithFormat:@"Listening on %@ (%@, on-device)…", phone ? @"phone" : @"glasses", language]}];
|
||||
// Bound a dictation to the recognizer's supported short-session workflow.
|
||||
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 55 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -85,6 +109,77 @@
|
||||
});
|
||||
return @"";
|
||||
}
|
||||
- (NSString *)startPhoneAudio {
|
||||
AVAudioSession *session = AVAudioSession.sharedInstance;
|
||||
NSError *error = nil;
|
||||
// Leave preferredInput unset so iOS selects its current default route,
|
||||
// including a headset microphone when one is attached.
|
||||
if (![session setCategory:AVAudioSessionCategoryPlayAndRecord mode:AVAudioSessionModeMeasurement
|
||||
options:AVAudioSessionCategoryOptionDefaultToSpeaker | AVAudioSessionCategoryOptionAllowBluetoothHFP error:&error])
|
||||
return [NSString stringWithFormat:@"Could not configure phone microphone: %@", error.localizedDescription];
|
||||
if (![session setActive:YES error:&error])
|
||||
return [NSString stringWithFormat:@"Could not activate phone microphone: %@", error.localizedDescription];
|
||||
self.phoneSessionActive = YES;
|
||||
self.phoneEngine = [AVAudioEngine new];
|
||||
AVAudioInputNode *input = self.phoneEngine.inputNode;
|
||||
AVAudioFormat *format = [input outputFormatForBus:0];
|
||||
if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again.";
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
[input installTapOnBus:0 bufferSize:1024 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) {
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
if (!owner || owner.generation != generation || !owner.accepting || !buffer.frameLength) return;
|
||||
// The tap owns buffer only for this callback. Copy before handing it
|
||||
// to the serial recognition queue, which finish drains before endAudio.
|
||||
AVAudioPCMBuffer *copy = [[AVAudioPCMBuffer alloc] initWithPCMFormat:buffer.format frameCapacity:buffer.frameLength];
|
||||
copy.frameLength = buffer.frameLength;
|
||||
const AudioBufferList *source = buffer.audioBufferList;
|
||||
AudioBufferList *destination = copy.mutableAudioBufferList;
|
||||
for (UInt32 i = 0; i < source->mNumberBuffers; i++)
|
||||
memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize);
|
||||
dispatch_async(owner.audioQueue, ^{ @autoreleasepool {
|
||||
if (owner.generation != generation) return;
|
||||
[request appendAudioPCMBuffer:copy];
|
||||
double sum = 0;
|
||||
if (copy.floatChannelData) {
|
||||
const float *samples = copy.floatChannelData[0];
|
||||
NSUInteger stride = copy.format.interleaved ? copy.format.channelCount : 1;
|
||||
for (NSUInteger i = 0; i < copy.frameLength; i++) sum += samples[i * stride] * samples[i * stride];
|
||||
}
|
||||
double rms = sqrt(sum / copy.frameLength);
|
||||
// The endpoint detector uses a 16 kHz sample clock; device input
|
||||
// commonly arrives at 48 kHz. Recognition accepts its native rate.
|
||||
int32_t samples = (int32_t)llround(copy.frameLength * 16000.0 / copy.format.sampleRate);
|
||||
BOOL ended = [detector acceptLevelRms:rms * 32768.0 sampleCount:samples];
|
||||
if (ended) dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (owner.generation == generation && owner.accepting) [owner finish];
|
||||
});
|
||||
}});
|
||||
}];
|
||||
self.phoneTapInstalled = YES;
|
||||
[self.phoneEngine prepare];
|
||||
if (![self.phoneEngine startAndReturnError:&error])
|
||||
return [NSString stringWithFormat:@"Could not start phone microphone: %@", error.localizedDescription];
|
||||
self.interruptionObserver = [NSNotificationCenter.defaultCenter addObserverForName:AVAudioSessionInterruptionNotification
|
||||
object:session queue:NSOperationQueue.mainQueue usingBlock:^(NSNotification *notification) {
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
if (owner.generation == generation && [notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue] == AVAudioSessionInterruptionTypeBegan)
|
||||
[owner complete:@"Phone microphone interrupted. Start voice input again to continue."];
|
||||
}];
|
||||
return @"";
|
||||
}
|
||||
- (void)stopPhoneAudio {
|
||||
if (self.interruptionObserver) [NSNotificationCenter.defaultCenter removeObserver:self.interruptionObserver];
|
||||
self.interruptionObserver = nil;
|
||||
[self.phoneEngine stop];
|
||||
if (self.phoneTapInstalled) [self.phoneEngine.inputNode removeTapOnBus:0];
|
||||
self.phoneTapInstalled = NO;
|
||||
self.phoneEngine = nil;
|
||||
if (self.phoneSessionActive) [AVAudioSession.sharedInstance setActive:NO withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation error:nil];
|
||||
self.phoneSessionActive = NO;
|
||||
}
|
||||
- (void)acceptPacket:(NSData *)packet {
|
||||
if (!self.accepting) return;
|
||||
if (self.pendingPackets >= 40) { [self complete:@"Audio processing fell behind. Please try again."]; return; }
|
||||
@@ -125,6 +220,7 @@
|
||||
- (void)finish {
|
||||
if (!self.accepting) return;
|
||||
self.accepting = NO;
|
||||
[self stopPhoneAudio];
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
[self emit:@{@"kind":@"finishing"}];
|
||||
@@ -146,6 +242,7 @@
|
||||
- (void)cancel {
|
||||
self.generation++;
|
||||
self.accepting = NO; self.pendingPackets = 0;
|
||||
[self stopPhoneAudio];
|
||||
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
|
||||
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user