forked from Blendman974/faceclaw
iOS TTF support; voice input
This commit is contained in:
1 parent
ae14edcb59
commit
b07ae0b479
67 files changed
+15717
-55
No files matched your search
@@ -55,6 +55,21 @@ test('iOS worker frames preserve baked grayscale bytes through the message bound
|
||||
assert.deepEqual(Buffer.from(messages[0].pixels, 'base64'), Buffer.from(pixels));
|
||||
});
|
||||
|
||||
test('settings-driven repaint runs after font cache invalidation, regardless of subscription order', () => {
|
||||
const listeners = [], tasks = [], painted = [];
|
||||
let cachedFont = 'Light';
|
||||
const api = load('app/ui/dashboard-settings.ts', {
|
||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||
: { Layer: class {}, ASSISTANT_MODEL_VALUES: [] },
|
||||
setTimeout: fn => tasks.push(fn),
|
||||
});
|
||||
api.onAnySettingChanged(() => painted.push(cachedFont));
|
||||
listeners.push(() => { cachedFont = 'Bold'; });
|
||||
for (const listener of listeners) listener('display.uiFont2');
|
||||
while (tasks.length) tasks.shift()();
|
||||
assert.deepEqual(painted, ['Bold']);
|
||||
});
|
||||
|
||||
test('background glasses input still composites frames; phone resume preserves the session; explicit stop stays stopped', async () => {
|
||||
const tasks = new Map(), screenStates = [], inputs = [], frames = [], previews = [], states = [];
|
||||
let nextTask = 0, starts = 0, stops = 0, session;
|
||||
@@ -75,6 +90,7 @@ test('background glasses input still composites frames; phone resume preserves t
|
||||
const settings = { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
|
||||
const modules = {
|
||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
|
||||
'../native/ios-bluetooth': { iosBluetooth: () => ({}) }, './glasses-session': { GlassesSession: Session },
|
||||
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
|
||||
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
|
||||
|
||||
@@ -222,3 +222,37 @@ test('a BLE wake services overdue heartbeat and lease without waiting for a susp
|
||||
assert.equal(h.transport.sent.length, count);
|
||||
assert.equal(h.session.state.phase, 'disconnected');
|
||||
});
|
||||
|
||||
test('glasses mic uses Android wire control and only routes bound-arm audio during capture', async t => {
|
||||
assert.equal(hex(p.audioControl(101, true)), '080f10659201020801');
|
||||
assert.equal(hex(p.audioControl(101, false)), '080f10659201020800');
|
||||
const h = harness(t), audio = [];
|
||||
await h.session.start(addresses);
|
||||
const send = id => h.transport.emit({ kind: 'notification', identifier: id, characteristic: p.G2_RENDER_NOTIFY, data: '010203' });
|
||||
send('R'); assert.equal(audio.length, 0);
|
||||
await h.session.setMicrophone(true, bytes => audio.push(hex(bytes)));
|
||||
assert.ok(h.transport.subscriptions.some(([id, c]) => id === 'R' && c === p.G2_RENDER_NOTIFY));
|
||||
assert.ok(h.transport.subscriptions.some(([id, c]) => id === 'L' && c === p.G2_RENDER_NOTIFY));
|
||||
send('stranger'); send('ring'); send('L'); send('R');
|
||||
assert.deepEqual(audio, ['010203', '010203']);
|
||||
const disabling = h.session.setMicrophone(false);
|
||||
send('R'); assert.equal(audio.length, 2); // Immediate, before the disable ACK.
|
||||
await disabling;
|
||||
assert.deepEqual(h.transport.sent.filter(s => s.message.command === 15).map(s =>
|
||||
[s.id, p.readInteger(p.readBytes(s.message.payload, 18), 1)]), [['R', 1], ['R', 0]]);
|
||||
});
|
||||
test('release while microphone enable awaits ACK sends disable last and rejects further audio', async t => {
|
||||
const h = harness(t); await h.session.start(addresses);
|
||||
h.transport.hold = true;
|
||||
let packets = 0;
|
||||
const enabling = h.session.setMicrophone(true, () => packets++);
|
||||
await until(() => h.transport.held.some(s => s.message.command === 15));
|
||||
const disabling = h.session.setMicrophone(false);
|
||||
h.transport.emit({ kind: 'notification', identifier: 'R', characteristic: p.G2_RENDER_NOTIFY, data: '01' });
|
||||
assert.equal(packets, 0);
|
||||
h.transport.hold = false;
|
||||
for (const s of h.transport.held) h.transport.ack(s.id, s.message);
|
||||
await Promise.all([enabling, disabling]);
|
||||
const mic = h.transport.sent.filter(s => s.message.command === 15);
|
||||
assert.equal(p.readInteger(p.readBytes(mic.at(-1).message.payload, 18), 1), 0);
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
// Runs the production Core Text renderer on macOS, without UIKit or a simulator.
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const { mkdtempSync } = require('node:fs');
|
||||
const { tmpdir } = require('node:os');
|
||||
const { join, resolve } = require('node:path');
|
||||
const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-font-'));
|
||||
execFileSync('xcrun', ['clang', '-fobjc-arc', '-Wall', '-Wextra', '-framework', 'Foundation', '-framework', 'CoreText', '-framework', 'CoreGraphics',
|
||||
'-I', 'App_Resources/iOS/src', 'tests/native/ios-font-test.m', 'App_Resources/iOS/src/FaceclawFontRenderer.m', '-o', join(dir, 'font-test')], { cwd: root, stdio: 'inherit' });
|
||||
execFileSync(join(dir, 'font-test'), [join(root, 'app/fonts/ttf'), dir], { stdio: 'inherit' });
|
||||
console.log(`Font test artifacts: ${dir}`);
|
||||
@@ -0,0 +1,11 @@
|
||||
// Exercise the actual decoder with stock G2 packet framing and encoded PCM.
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const { mkdtempSync, readdirSync } = require('node:fs');
|
||||
const { tmpdir } = require('node:os');
|
||||
const { join, resolve } = require('node:path');
|
||||
const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-lc3-'));
|
||||
const src = 'App_Resources/iOS/src';
|
||||
const files = readdirSync(join(root, src, 'lc3')).filter(f => f.endsWith('.c')).map(f => `${src}/lc3/${f}`);
|
||||
execFileSync('xcrun', ['clang', '-fobjc-arc', '-O2', '-framework', 'Foundation', '-I', src, '-I', `${src}/lc3`,
|
||||
'tests/native/ios-lc3-test.m', `${src}/FaceclawLc3Decoder.m`, ...files, '-o', join(dir, 'lc3-test')], { cwd: root, stdio: 'inherit' });
|
||||
execFileSync(join(dir, 'lc3-test'), [], { stdio: 'inherit' });
|
||||
@@ -0,0 +1,63 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
function harness() {
|
||||
const native = { start: () => '', cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [];
|
||||
let authorization = 3, permissionRequests = 0, release;
|
||||
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
|
||||
requestAuthorization: done => { permissionRequests++; done(3); } },
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b } };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
bridge.onTranscript(event => text.push({ ...event }));
|
||||
const session = { setMicrophone(enabled, fn) {
|
||||
commands.push(enabled);
|
||||
if (enabled) { this.packet = fn; return new Promise(resolve => { release = resolve; }); }
|
||||
return Promise.resolve();
|
||||
} };
|
||||
return { bridge, native, session, audio, commands, text, timers, release: () => release(),
|
||||
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
|
||||
event: event => native.eventHandler(JSON.stringify(event)) };
|
||||
}
|
||||
test('iOS speech permission is requested only in foreground and denial stays actionable', async () => {
|
||||
const h = harness(); h.authorization(0);
|
||||
assert.equal(await h.bridge.prepare(false), false); assert.equal(h.permissionRequests(), 0);
|
||||
assert.equal(await h.bridge.prepare(true), true); assert.equal(h.permissionRequests(), 1);
|
||||
h.authorization(2); assert.equal(await h.bridge.prepare(true), false);
|
||||
assert.match(h.bridge.statusText, /Settings/);
|
||||
});
|
||||
test('glasses speech sends PCM-source packets, replaces partials, and drains final text after manual release', async () => {
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {});
|
||||
h.session.packet(new Uint8Array([1, 2, 3])); assert.deepEqual([...h.audio[0]], [1, 2, 3]);
|
||||
h.release(); await start;
|
||||
h.event({ kind: 'transcript', text: 'hello', final: false });
|
||||
h.event({ kind: 'transcript', text: 'hello world', final: false });
|
||||
h.bridge.stopPushToTalk(); assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
|
||||
h.session.packet(new Uint8Array([4])); assert.equal(h.audio.length, 1);
|
||||
h.event({ kind: 'transcript', text: 'Hello world.', final: true });
|
||||
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||
assert.deepEqual(h.text, [{ text: 'hello', isFinal: false }, { text: 'hello world', isFinal: false }, { text: 'Hello world.', isFinal: true }]);
|
||||
h.event({ kind: 'transcript', text: 'stale', final: true }); assert.equal(h.text.length, 3);
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
test('cancel and disconnect during enable cannot resurrect capture or leave its microphone listener active', async () => {
|
||||
for (const stop of ['stop', 'handleSessionEnded']) {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
const start = h.bridge.startGlassesCapture(h.session, () => {});
|
||||
h.bridge[stop](); h.release(); await start;
|
||||
h.session.packet(new Uint8Array([9]));
|
||||
h.event({ kind: 'transcript', text: 'stale', final: true });
|
||||
assert.deepEqual(h.commands, [true, false]); assert.equal(h.audio.length, 0); assert.equal(h.text.length, 0);
|
||||
assert.equal(h.timers.size, 0); assert.equal(ends, stop === 'stop' ? 0 : 1);
|
||||
}
|
||||
});
|
||||
test('missing microphone data ends the session and disables the mic with an actionable status', async () => {
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start;
|
||||
for (const fn of h.timers.values()) fn();
|
||||
assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/);
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "FaceclawFontRenderer.h"
|
||||
#import <math.h>
|
||||
static int U16(const uint8_t *p) { return p[0] | p[1] << 8; }
|
||||
static int S16(const uint8_t *p) { return (int16_t)U16(p); }
|
||||
static void Check(BOOL ok, NSString *message) { if (!ok) { NSLog(@"FAIL: %@", message); exit(1); } }
|
||||
static void SavePGM(NSData *packet, NSString *path) {
|
||||
const uint8_t *p = packet.bytes;
|
||||
NSMutableData *out = [[NSString stringWithFormat:@"P5\n%d %d\n255\n", U16(p), U16(p + 2)] dataUsingEncoding:NSUTF8StringEncoding].mutableCopy;
|
||||
[out appendBytes:p + 4 length:packet.length - 4]; [out writeToFile:path atomically:YES];
|
||||
}
|
||||
int main(int argc, char **argv) { @autoreleasepool {
|
||||
Check(argc == 3, @"font directory and output directory required");
|
||||
NSString *dir = @(argv[1]), *outDir = @(argv[2]);
|
||||
NSString *font = [dir stringByAppendingPathComponent:@"Roboto-Regular.ttf"];
|
||||
NSString *bad = [outDir stringByAppendingPathComponent:@"invalid.ttf"];
|
||||
[@"not a font" writeToFile:bad atomically:YES encoding:NSUTF8StringEncoding error:nil];
|
||||
Check(![FaceclawFontRenderer canLoadFont:bad], @"invalid font rejected");
|
||||
Check(![FaceclawFontRenderer canLoadFont:@"/missing/font.ttf"], @"missing font rejected");
|
||||
Check([FaceclawFontRenderer renderGlyphCell:font size:NAN codePoint:65 gamma:1].length == 0, @"invalid size rejected");
|
||||
Check([FaceclawFontRenderer renderGlyphCell:font size:16 codePoint:0xd800 gamma:1].length == 0, @"surrogate rejected");
|
||||
NSMutableArray *metadata = [NSMutableArray new];
|
||||
for (NSString *file in [[NSFileManager defaultManager] contentsOfDirectoryAtPath:dir error:nil]) {
|
||||
if (![file.pathExtension isEqual:@"ttf"]) continue;
|
||||
NSString *path = [dir stringByAppendingPathComponent:file];
|
||||
Check([FaceclawFontRenderer canLoadFont:path], file);
|
||||
NSString *name = [FaceclawFontRenderer getFontName:path];
|
||||
Check(name.length > 2, @"family/style metadata");
|
||||
[metadata addObject:@{@"file":file, @"name":name, @"metrics14":[FaceclawFontRenderer getFontMetrics:path size:14]}];
|
||||
for (NSNumber *size in @[@8, @14, @18, @32, @64]) {
|
||||
NSData *glyph = [FaceclawFontRenderer renderGlyphCell:path size:size.doubleValue codePoint:65 gamma:1];
|
||||
Check(glyph.length > 10, @"bundled glyph coverage");
|
||||
const uint8_t *p = glyph.bytes;
|
||||
Check(glyph.length == (NSUInteger)(10 + U16(p+6)*U16(p+8)), @"glyph packet dimensions");
|
||||
}
|
||||
}
|
||||
Check(metadata.count == 12, @"all twelve bundled fonts");
|
||||
NSString *mono = [dir stringByAppendingPathComponent:@"RobotoMono-Regular.ttf"];
|
||||
Check(fabs([FaceclawFontRenderer measureTextExact:mono text:@"i" size:48] - [FaceclawFontRenderer measureTextExact:mono text:@"W" size:48]) < 0.01, @"monospace widths");
|
||||
double av = [FaceclawFontRenderer measureTextExact:font text:@"AV" size:32];
|
||||
double a = [FaceclawFontRenderer measureTextExact:font text:@"A" size:32], v = [FaceclawFontRenderer measureTextExact:font text:@"V" size:32];
|
||||
Check(av < a + v, @"kerning retained");
|
||||
NSMutableDictionary *packets = [NSMutableDictionary new];
|
||||
NSArray *metrics = [[FaceclawFontRenderer getFontMetrics:font size:32] componentsSeparatedByString:@" "];
|
||||
NSInteger ascent = [metrics[0] integerValue], descent = [metrics[1] integerValue];
|
||||
for (NSString *text in @[@"A", @"g", @"j", @"É", @" "]) {
|
||||
NSData *glyph = [FaceclawFontRenderer renderGlyphCell:font size:32 codePoint:[text characterAtIndex:0] gamma:1];
|
||||
const uint8_t *g = glyph.bytes;
|
||||
Check(fabs(U16(g)/64.0 - [FaceclawFontRenderer measureTextExact:font text:text size:32]) <= 1.0/128, @"advance precision");
|
||||
NSData *line = [FaceclawFontRenderer renderText:font text:text size:32 gamma:1]; const uint8_t *p = line.bytes;
|
||||
if ([text isEqual:@" "]) { Check(glyph.length == 10 && U16(g) > 0, @"space preserves advance with no ink"); continue; }
|
||||
Check(S16(g+4) >= 0 && S16(g+4)+U16(g+8) <= ascent+descent, @"glyph inside line box");
|
||||
NSInteger shiftX = MAX(0, -S16(g+2));
|
||||
// Glyph bearings/top must place precisely the same pixels as a native
|
||||
// single-character line. Detects upside-down buffers and baseline drift.
|
||||
for (NSInteger y=0; y<U16(g+8); y++) for (NSInteger x=0; x<U16(g+6); x++) {
|
||||
NSInteger targetX = shiftX+S16(g+2)+x, targetY = S16(g+4)+y;
|
||||
Check(targetX < U16(p) && targetY < U16(p+2), @"glyph placement bounds");
|
||||
Check(g[10+y*U16(g+6)+x] == p[4+targetY*U16(p)+targetX], @"glyph and line coverage agree");
|
||||
}
|
||||
packets[text] = [glyph base64EncodedStringWithOptions:0];
|
||||
}
|
||||
NSData *light = [FaceclawFontRenderer renderGlyphCell:font size:32 codePoint:65 gamma:0.5];
|
||||
NSData *dark = [FaceclawFontRenderer renderGlyphCell:font size:32 codePoint:65 gamma:1.6];
|
||||
BOOL antialiased = NO; const uint8_t *l = light.bytes, *d = dark.bytes;
|
||||
for (NSUInteger i=10; i<light.length; i++) { Check(l[i] >= d[i], @"gamma monotonic"); if (l[i] > 0 && l[i] < 255) antialiased = YES; }
|
||||
Check(antialiased, @"antialias coverage");
|
||||
Check([FaceclawFontRenderer renderGlyphCell:font size:32 codePoint:0x1f642 gamma:1].length > 10, @"non-BMP fallback glyph");
|
||||
NSString *sample = @"AgjÉ 0123456789 — AV fi ffi";
|
||||
SavePGM([FaceclawFontRenderer renderText:font text:sample size:32 gamma:1], [outDir stringByAppendingPathComponent:@"line.pgm"]);
|
||||
NSString *paragraph = @"The quick brown fox jumps over the lazy dog.\nCafé déjà vu — 23°C. ALongWordWithoutSpacesToExerciseWrapping. More text for truncation.";
|
||||
NSData *wrapped = [FaceclawFontRenderer renderWrapped:font text:paragraph size:18 width:200 gamma:1 maxLines:3];
|
||||
Check(U16(wrapped.bytes) == 200 && U16((const uint8_t *)wrapped.bytes+2) <= 3*24, @"bounded wrapped text");
|
||||
SavePGM(wrapped, [outDir stringByAppendingPathComponent:@"wrapped.pgm"]);
|
||||
NSData *reference = [FaceclawFontRenderer renderGlyphCell:font size:14 codePoint:65 gamma:1];
|
||||
dispatch_apply(32, dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^(size_t i) { (void)i; @autoreleasepool {
|
||||
Check([[FaceclawFontRenderer renderGlyphCell:font size:14 codePoint:65 gamma:1] isEqual:reference], @"concurrent workers deterministic");
|
||||
}});
|
||||
NSDictionary *result = @{@"fonts":metadata, @"glyphs32":packets, @"kernAV32":@(av-a-v)};
|
||||
[[NSJSONSerialization dataWithJSONObject:result options:NSJSONWritingPrettyPrinted error:nil] writeToFile:[outDir stringByAppendingPathComponent:@"result.json"] atomically:YES];
|
||||
NSLog(@"PASS: font loading, metadata, glyph placement, coverage, kerning, wrapping, gamma and worker concurrency");
|
||||
} return 0; }
|
||||
@@ -0,0 +1,34 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "FaceclawLc3Decoder.h"
|
||||
#include "lc3/lc3.h"
|
||||
#include <math.h>
|
||||
#include <assert.h>
|
||||
int main(void) { @autoreleasepool {
|
||||
void *memory = calloc(1, lc3_encoder_size(10000, 16000));
|
||||
lc3_encoder_t encoder = lc3_setup_encoder(10000, 16000, 0, memory);
|
||||
assert(encoder);
|
||||
FaceclawLc3Decoder *decoder = [FaceclawLc3Decoder new];
|
||||
double energy = 0; NSUInteger samples = 0;
|
||||
for (int packetIndex = 0; packetIndex < 32; packetIndex++) {
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:205];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
for (int frame = 0; frame < 5; frame++) {
|
||||
int16_t input[160];
|
||||
for (int i = 0; i < 160; i++) input[i] = (int16_t)(12000 * sin(2 * M_PI * 440 * (packetIndex * 800 + frame * 160 + i) / 16000));
|
||||
assert(lc3_encode(encoder, LC3_PCM_FORMAT_S16, input, 1, 40, bytes + frame * 40) == 0);
|
||||
}
|
||||
bytes[204] = (uint8_t)(250 + packetIndex); // Cross counter wrap.
|
||||
NSData *pcm = [decoder decodePacket:packet];
|
||||
assert(pcm.length == 1600);
|
||||
const int16_t *out = pcm.bytes;
|
||||
for (int i = 0; i < 800; i++) { energy += (double)out[i] * out[i]; samples++; }
|
||||
assert([decoder decodePacket:packet] == nil); // Other arm's relay.
|
||||
bytes[204]--; assert([decoder decodePacket:packet] == nil); // Late relay.
|
||||
}
|
||||
assert(decoder.packets == 32 && decoder.duplicates == 64 && decoder.missing == 0);
|
||||
assert(sqrt(energy / samples) > 6000 && sqrt(energy / samples) < 10000);
|
||||
assert([decoder decodePacket:[NSMutableData dataWithLength:204]] == nil);
|
||||
assert(decoder.errors == 1);
|
||||
free(memory);
|
||||
puts("PASS: LC3 G2 packets decode to 16 kHz PCM; signal energy, duplicate/late relays, counter wrap and malformed packets checked.");
|
||||
} return 0; }
|
||||
Reference in new issue
Block a user