iOS TTF support; voice input

This commit is contained in:
James Babcock committed 2026-09-07 16:57:49 -07:00
1 parent ae14edcb59
commit b07ae0b479
67 files changed
+15717 -55

No files matched your search

+2 -2
View File
@@ -45,7 +45,7 @@ export type FilesAppOptions = InProcessAppOptions & {
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
let created: InProcessWindow | null = null;
const browser = new FileBrowserLayer({
isSupportedFile: (name) => TEXT_FILE.test(name) || (!global.isIOS && (isDecodableImageFile(name) || EHPK_FILE.test(name) || isFontFile(name))),
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || (!global.isIOS && (isDecodableImageFile(name) || EHPK_FILE.test(name))),
// The browser handles double-click itself (up a level), so it is not
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
onLeave: () => shell.yieldFocusToSidebar(),
@@ -170,7 +170,7 @@ function confirmEhpkPermissions(ctx: LayerContext, entry: DirectoryEntry, procee
* window for viewable types, empty for everything else (metadata only).
*/
function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileInfoAction[] {
if (global.isIOS && !TEXT_FILE.test(entry.name)) return [];
if (global.isIOS && !TEXT_FILE.test(entry.name) && !isFontFile(entry.name)) return [];
if (TEXT_FILE.test(entry.name)) {
return [
{
+1 -1
View File
@@ -27,7 +27,7 @@ const PARAGRAPH =
/**
* Font previewer for a font file on disk (TTF/OTF/TTC), rendered through the
* Android text stack: a single-line pangram, a character-set sample, and a
* native text stack: a single-line pangram, a character-set sample, and a
* wrapped paragraph, all at an adjustable pixel size. Scroll changes size,
* click cycles the coverage gamma (for judging antialiasing on the lens),
* double-click closes. Sized to its hosting stack (a Files document window,
+1
View File
@@ -21,6 +21,7 @@ const wrap = (command: number, magic: number, field: number, body: Uint8Array) =
export const authentication = (magic: number) => wrap(4, magic, 3, concat(integer(1, 1), integer(2, 4)))
export const prelude = () => wrap(2, 156, 4, bytes(3, bytes(2, bytes(2, concat(integer(1, 0), integer(2, 0))))))
export const heartbeat = (magic: number) => wrap(12, magic, 14, integer(1, 0))
export const audioControl = (magic: number, enabled: boolean) => wrap(15, magic, 18, integer(1, enabled ? 1 : 0))
export const settingsQuery = (magic: number) => wrap(2, magic, 4, integer(1, 1))
export const shutdown = (magic: number) => wrap(9, magic, 11, integer(1, 1))
export const framebufferLease = (acquire: boolean) => concat(integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, acquire ? 5 : 6, 0, 0])))
+41
View File
@@ -51,6 +51,10 @@ export class GlassesSession {
private layoutCreated = false
private off: () => void
private stopping: Promise<void> | null = null
private audioListener: ((packet: Uint8Array) => void) | null = null
private microphoneToken = 0
private microphoneWork: Promise<void> = Promise.resolve()
private microphoneEnabled = false
constructor(private readonly transport: SessionTransport,
private readonly deflate: (data: Uint8Array) => Uint8Array,
private readonly onState: (state: SessionState) => void,
@@ -211,6 +215,13 @@ export class GlassesSession {
return
}
if (event.kind !== 'notification' || !event.data) return
if (event.characteristic === protocol.G2_RENDER_NOTIFY && event.identifier !== this.ids.ring) {
if (this.audioListener && this.state.phase === 'connected') {
this.audioListener(hexToBytes(event.data))
this.onActivity(); this.wake()
}
return
}
const ring = event.identifier === this.ids.ring
if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return
try {
@@ -268,6 +279,30 @@ export class GlassesSession {
if (this.state.phase !== 'connected') return
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
}
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
const token = ++this.microphoneToken, generation = this.generation
this.audioListener = enabled ? listener ?? null : null
const current = () => generation === this.generation && token === this.microphoneToken
const work = this.microphoneWork.catch(() => {}).then(async () => {
if (!current()) return
if (this.state.phase !== 'connected') {
if (enabled) throw new Error('Connect the glasses before starting voice input.')
return
}
if (enabled) {
await Promise.all(['left', 'right'].map(role => this.transport.subscribe(this.ids[role], protocol.G2_RENDER_NOTIFY)))
if (!current()) return
}
await this.request('right', protocol.SID.hub, magic => protocol.audioControl(magic, enabled), enabled ? 'Microphone enable' : 'Microphone disable')
if (current()) { this.microphoneEnabled = enabled; this.log(`Glasses microphone ${enabled ? 'enabled' : 'disabled'}`) }
}).catch(error => {
if (current()) { this.audioListener = null; this.microphoneEnabled = false }
throw error
})
this.microphoneWork = work
return work
}
wake(): void {
if (this.timer !== null) { clearTimeout(this.timer); this.timer = null }
void this.pump()
@@ -308,6 +343,7 @@ export class GlassesSession {
}
private reset(): void {
++this.generation
++this.microphoneToken; this.audioListener = null; this.microphoneEnabled = false
if (this.timer !== null) clearTimeout(this.timer)
if (this.retryTimer !== null) clearTimeout(this.retryTimer)
if (this.ringRetryTimer !== null) clearTimeout(this.ringRetryTimer)
@@ -337,6 +373,11 @@ export class GlassesSession {
}
private async stopInternal(): Promise<void> {
this.wanted = false
// Stop mic capture before ending the display session. Late enable ACKs
// cannot leave a microphone streaming after an explicit Disconnect.
if (this.audioListener || this.microphoneEnabled) {
try { await this.setMicrophone(false) } catch (error) { this.log(`Microphone cleanup: ${this.message(error)}`) }
}
const cleanup = this.layoutCreated && this.state.phase === 'connected'
this.update('disconnecting', 'Disconnecting…'); this.reset()
try {
+21 -3
View File
@@ -1,5 +1,6 @@
import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescript/core'
import { iosBluetooth } from '../native/ios-bluetooth'
import { iosVoiceInput } from '../native/ios-voice-input'
import { GlassesSession, type SessionState } from './glasses-session'
import { loadDeviceAddresses } from './device-addresses'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -47,7 +48,8 @@ export class IosPreviewController {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
disconnect: () => this.disconnect(),
startVoiceCapture: () => this.typeIntoApp(),
startVoiceCapture: () => this.startVoiceCapture(),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
startTextSettingEdit: setting => this.editSetting(setting),
startTextSettingsEdit: async (settings, _title, finished) => {
@@ -62,7 +64,8 @@ export class IosPreviewController {
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
shell.configure({
actions: this.actions,
voiceInputEnabled: false,
voiceInputEnabled: true,
prepareVoiceCapture: () => this.prepareVoiceCapture(),
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
@@ -285,7 +288,7 @@ export class IosPreviewController {
openSettings: section => { void this.launchApp('settings', { section }) },
startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) },
endTextSettingEdit: () => {},
startTextInput: () => { void this.typeIntoApp() },
startTextInput: () => shell.startVoiceInput(),
})
this.appHosts.set(appId, host)
return host
@@ -307,6 +310,7 @@ export class IosPreviewController {
if (!this.session) {
const { deflate } = require('pako') as { deflate: (data: Uint8Array) => Uint8Array }
this.session = new GlassesSession(iosBluetooth(), deflate, state => {
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
this.syncRuntime()
if (this.active) this.onConnectionState(state)
@@ -322,6 +326,20 @@ export class IosPreviewController {
await this.session.start(addresses)
}
async disconnect(): Promise<void> { await this.session?.stop() }
startVoiceInput(): void { shell.startVoiceInput() }
private async prepareVoiceCapture(): Promise<boolean> {
if (this.session?.state.phase !== 'connected') {
if (this.active) this.onError('Connect the glasses to use their microphone.')
return false
}
const ready = await iosVoiceInput.prepare(this.active)
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && this.session?.state.phase === 'connected'
}
private async startVoiceCapture(): Promise<void> {
if (!this.session || this.session.state.phase !== 'connected') return
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
+15 -7
View File
@@ -4,14 +4,14 @@
* app assets on first listing (so the directory is the single source of
* truth), and user installs from the Files app ("Install" on a font file).
*
* Listing parses each file's name table for family/style (via the Java
* FontFileRenderer) and probes monospace-ness by comparing advances, both
* Listing parses each file's name table for family/style (via the native
* font renderer) and probes monospace-ness by comparing advances, both
* memoized per path for the session.
*/
import { File, Folder, knownFolders, path as nsPath } from "@nativescript/core";
import { canLoadFontFile, fontFileDisplayName, isFontFile } from "../native/font-files";
declare const com: any;
import { fontRenderer } from "../native/font-renderer";
declare const java: any;
declare const global: any;
@@ -62,7 +62,7 @@ let preinstalledEnsured = false;
* before every listing.
*/
export function ensurePreinstalledFonts(): void {
if (preinstalledEnsured || !global.isAndroid) return;
if (preinstalledEnsured || (!global.isAndroid && !global.isIOS)) return;
const assetsDir = nsPath.join(knownFolders.currentApp().path, "fonts/ttf");
for (const fileName of PREINSTALLED_FONT_FILES) {
const target = installedFontPath(fileName);
@@ -83,7 +83,7 @@ export function ensurePreinstalledFonts(): void {
* same-named file). Returns an error message for the UI, or null on success.
*/
export function installFontFile(sourcePath: string): string | null {
if (!global.isAndroid) return "Not available on this platform.";
if ((!global.isAndroid && !global.isIOS)) return "Not available on this platform.";
if (!isFontFile(sourcePath)) return "Not a font file.";
if (!canLoadFontFile(sourcePath)) return "This file could not be loaded as a font.";
const fileName = sourcePath.slice(sourcePath.lastIndexOf("/") + 1);
@@ -108,7 +108,7 @@ export function isFontInstalled(sourcePath: string): boolean {
* then weight. Unloadable files are skipped.
*/
export function listInstalledFonts(): InstalledFont[] {
if (!global.isAndroid) return [];
if ((!global.isAndroid && !global.isIOS)) return [];
ensurePreinstalledFonts();
const fonts: InstalledFont[] = [];
try {
@@ -195,7 +195,7 @@ function styleWeightOrder(style: string): number {
function isMonospaceFont(path: string): boolean {
try {
const measure = (text: string) =>
Number(com.faceclaw.app.FontFileRenderer.measureTextExact(path, text, 48));
Number(fontRenderer.measureTextExact(path, text, 48));
const narrow = measure("i");
if (!(narrow > 0)) return false;
return (
@@ -217,6 +217,14 @@ function isMonospaceFont(path: string): boolean {
// isolates on first launch (each resolving the default UI font), and readers
// must never see a half-written font file.
function copyFileBytes(source: string, target: string): void {
if (source === target) return;
if (global.isIOS) {
const data = NSData.dataWithContentsOfFile(source);
// Foundation's atomic write uses a sibling temporary file and rename.
// Multiple worker isolates can safely preinstall the same bundled face.
if (!data || !data.writeToFileAtomically(target, true)) throw new Error(`Copy to ${target} failed`);
return;
}
const temp = `${target}.tmp-${Math.floor(Math.random() * 0x7fffffff).toString(36)}`;
const input = new java.io.FileInputStream(source);
try {
+10 -11
View File
@@ -1,7 +1,6 @@
/**
* TTF/OTF fonts for on-glasses UI text, rendered per-codepoint through the
* Java FontFileRenderer (Android minikin/HarfBuzz: real shaping, kerning,
* antialiasing) and cached as 4bpp coverage rasters that ride the
* native font renderer (Android Paint / iOS Core Text) and cached as 4bpp coverage rasters that ride the
* texture-cache glyph pipeline (CFW mode 14; see
* notes/texture-cache-display-list-design.md and graphics/glyph-wire.ts).
*
@@ -14,9 +13,9 @@
* never drifts.
*
* Kerning uses memoized per-pair measurements (measure("ab") - a - b);
* ligatures are disabled Java-side so per-codepoint rasters plus pairwise
* ligatures are disabled native-side so per-codepoint rasters plus pairwise
* kerning fully describe a line. Glyph rasters and advances come from one
* JNI render per (font, size, codepoint) per JS context, memoized forever.
* native render per (font, size, codepoint) per JS context, memoized forever.
*
* Cross-context determinism: worker threads and the main thread each build
* their own TtfFont, but the Java GlyphAtlas dedupes by atlasKey and treats
@@ -26,8 +25,8 @@
import { allocateFontFingerprintId, type Glyph } from "./bdffont";
import { GrayImage, grayToNibble, type GlyphFont } from "./image";
import { toUint8Array } from "../util/array-util";
import { fontRenderer } from "../native/font-renderer";
declare const com: any;
declare const global: any;
/**
@@ -65,7 +64,7 @@ export class TtfFont implements GlyphFont {
/**
* Load (or return the memoized) font at a pixel size. Null when the font
* file is missing/unloadable or off-Android — callers fall back to a BDF
* file is missing/unloadable or the platform has no renderer — callers fall back to a BDF
* face.
*/
static load(path: string, sizePx: number, gamma = DEFAULT_TTF_GAMMA): TtfFont | null {
@@ -73,9 +72,9 @@ export class TtfFont implements GlyphFont {
const cached = loadedFonts.get(key);
if (cached !== undefined) return cached;
let font: TtfFont | null = null;
if (global.isAndroid && sizePx > 0) {
if ((global.isAndroid || global.isIOS) && Number.isFinite(sizePx) && sizePx > 0) {
try {
const raw = String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, sizePx) ?? "");
const raw = String(fontRenderer.getFontMetrics(path, sizePx) ?? "");
const parts = raw.split(" ").map((part) => parseInt(part, 10));
if (parts.length === 3 && parts.every((part) => Number.isFinite(part))) {
const [ascent, descent] = parts as [number, number, number];
@@ -170,7 +169,7 @@ export class TtfFont implements GlyphFont {
/**
* Kerning between two adjacent codepoints: measure(ab) - a - b, memoized.
* One JNI measurement per distinct pair per JS context.
* One native measurement per distinct pair per JS context.
*/
private kernBetween(a: number, b: number): number {
const key = a * 0x200000 + b;
@@ -179,7 +178,7 @@ export class TtfFont implements GlyphFont {
let kern = 0;
try {
const pair = Number(
com.faceclaw.app.FontFileRenderer.measureTextExact(
fontRenderer.measureTextExact(
this.path,
String.fromCodePoint(a) + String.fromCodePoint(b),
this.sizePx,
@@ -203,7 +202,7 @@ export class TtfFont implements GlyphFont {
let bytes: Uint8Array;
try {
bytes = toUint8Array(
com.faceclaw.app.FontFileRenderer.renderGlyphCell(this.path, this.sizePx, codePoint, this.gamma),
fontRenderer.renderGlyphCell(this.path, this.sizePx, codePoint, this.gamma),
);
} catch (error) {
console.warn(`renderGlyphCell failed for U+${codePoint.toString(16)}: ${error}`);
+14 -15
View File
@@ -1,17 +1,16 @@
/**
* Font-file text rendering for the on-glasses font previewer, backed by the
* Java FontFileRenderer (Android Typeface/Paint, so shaping, kerning, and
* line breaking come from minikin/HarfBuzz). Rendered text arrives as
* native font renderer (Android Paint / iOS Core Text). Rendered text arrives as
* antialiased 8bpp coverage, which the compositor quantizes to the display's
* 16 shades.
*/
import { GrayImage } from "../graphics/image";
import { grayImageFromPacket } from "./image-files";
declare const com: any;
import { fontRenderer } from "./font-renderer";
declare const global: any;
/** Font file formats Android's Typeface can load. */
/** Font file formats supported by the native renderers. */
const FONT_FILE = /\.(ttf|otf|ttc)$/i;
export function isFontFile(name: string): boolean {
@@ -37,9 +36,9 @@ export const DEFAULT_FONT_GAMMA = 1.0;
/** Whether the font file parses as a loadable font. */
export function canLoadFontFile(path: string): boolean {
if (!global.isAndroid) return false;
if (!global.isAndroid && !global.isIOS) return false;
try {
return !!com.faceclaw.app.FontFileRenderer.canLoadFont(path);
return !!fontRenderer.canLoadFont(path);
} catch {
return false;
}
@@ -50,9 +49,9 @@ export function canLoadFontFile(path: string): boolean {
* unavailable (caller should fall back to the filename).
*/
export function fontFileDisplayName(path: string): { family: string; style: string } | null {
if (!global.isAndroid) return null;
if (!global.isAndroid && !global.isIOS) return null;
try {
const raw = String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? "");
const raw = String(fontRenderer.getFontName(path) ?? "");
if (!raw) return null;
const [family, style] = raw.split("\n");
if (!family) return null;
@@ -64,9 +63,9 @@ export function fontFileDisplayName(path: string): { family: string; style: stri
/** Line metrics at a pixel size; null when the font cannot be loaded. */
export function fontFileMetrics(path: string, sizePx: number): FontFileMetrics | null {
if (!global.isAndroid) return null;
if (!global.isAndroid && !global.isIOS) return null;
try {
const raw = String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, sizePx) ?? "");
const raw = String(fontRenderer.getFontMetrics(path, sizePx) ?? "");
const parts = raw.split(" ").map((part) => parseInt(part, 10));
if (parts.length !== 3 || parts.some((part) => !Number.isFinite(part))) return null;
return { ascent: parts[0]!, descent: parts[1]!, lineGap: parts[2]! };
@@ -86,9 +85,9 @@ export function renderFontFileText(
sizePx: number,
gamma = DEFAULT_FONT_GAMMA,
): GrayImage | null {
if (!global.isAndroid) return null;
if (!global.isAndroid && !global.isIOS) return null;
try {
return grayImageFromPacket(com.faceclaw.app.FontFileRenderer.renderText(path, text, sizePx, gamma));
return grayImageFromPacket(fontRenderer.renderText(path, text, sizePx, gamma));
} catch (error) {
console.warn(`renderFontFileText failed for ${path}: ${error}`);
return null;
@@ -96,7 +95,7 @@ export function renderFontFileText(
}
/**
* Render a paragraph wrapped to maxWidth (StaticLayout line breaking),
* Render a paragraph wrapped to maxWidth (native line breaking),
* truncated with an ellipsis past maxLines.
*/
export function renderFontFileWrapped(opts: {
@@ -107,10 +106,10 @@ export function renderFontFileWrapped(opts: {
maxLines: number;
gamma?: number;
}): GrayImage | null {
if (!global.isAndroid) return null;
if (!global.isAndroid && !global.isIOS) return null;
try {
return grayImageFromPacket(
com.faceclaw.app.FontFileRenderer.renderWrapped(
fontRenderer.renderWrapped(
opts.path,
opts.text,
opts.sizePx,
+13
View File
@@ -0,0 +1,13 @@
declare const FaceclawFontRenderer: any
function bytes(data: NSData | null): Uint8Array {
return data ? new Uint8Array(interop.bufferFromData(data)).slice() : new Uint8Array()
}
export const fontRenderer = {
canLoadFont: (path: string): boolean => !!FaceclawFontRenderer.canLoadFont(path),
getFontName: (path: string): string => String(FaceclawFontRenderer.getFontName(path) ?? ''),
getFontMetrics: (path: string, size: number): string => String(FaceclawFontRenderer.getFontMetricsSize(path, size) ?? ''),
measureTextExact: (path: string, text: string, size: number): number => Number(FaceclawFontRenderer.measureTextExactTextSize(path, text, size)),
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderGlyphCellSizeCodePointGamma(path, size, cp, gamma)),
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderTextTextSizeGamma(path, text, size, gamma)),
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => bytes(FaceclawFontRenderer.renderWrappedTextSizeWidthGammaMaxLines(path, text, size, width, gamma, lines)),
}
+12
View File
@@ -0,0 +1,12 @@
/** Platform boundary for font rasterization; layout and glyph caches stay shared. */
import { toUint8Array } from '../util/array-util'
declare const com: any
export const fontRenderer = {
canLoadFont: (path: string): boolean => !!com.faceclaw.app.FontFileRenderer.canLoadFont(path),
getFontName: (path: string): string => String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? ''),
getFontMetrics: (path: string, size: number): string => String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, size) ?? ''),
measureTextExact: (path: string, text: string, size: number): number => Number(com.faceclaw.app.FontFileRenderer.measureTextExact(path, text, size)),
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderGlyphCell(path, size, cp, gamma)),
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderText(path, text, size, gamma)),
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderWrapped(path, text, size, width, gamma, lines)),
}
+1
View File
@@ -0,0 +1 @@
export { voiceControlBridge as iosVoiceInput } from './voice-control.ios'
+116
View File
@@ -0,0 +1,116 @@
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
}
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* exclusively from the connected glasses, never from the phone microphone.
*/
export class IosVoiceControlBridge {
private native: any = null
private readonly statuses = new Set<(state: VoiceControlState) => void>()
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
private readonly ends = new Set<() => void>()
private status = 'Voice input ready.'
private generation = 0
private session: IosMicrophoneSession | null = null
private capturing = false
private finalizing = false
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status }); return () => { this.statuses.delete(listener) }
}
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
this.transcripts.add(listener); return () => { this.transcripts.delete(listener) }
}
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status }) }
private ensureNative(): any {
if (!this.native) {
this.native = FaceclawSpeech.new()
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
}
return this.native
}
async prepare(foreground: boolean): Promise<boolean> {
let status = FaceclawSpeech.authorizationStatus()
if (status === 0) {
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
}
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
this.stop()
const generation = ++this.generation
this.log = log; this.session = session; this.packets = 0; this.capturing = true
const error = String(this.ensureNative().start() ?? '')
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
try {
await session.setMicrophone(true, packet => {
if (generation !== this.generation || !this.capturing) return
this.packets++
const copy = new Uint8Array(packet)
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log('Voice: glasses microphone enabled; using on-device recognition')
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
this.stop(); this.setStatus('No microphone audio received. Reconnect the glasses and try again.'); this.emitEnd()
}
}, 5000)
} catch (error) {
if (generation !== this.generation) return
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
stopPushToTalk(): void {
if (!this.capturing) return
this.capturing = false; this.finalizing = true
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus('Recognizing…'); this.native?.finish()
}
stop(): void {
++this.generation
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
}
handleSessionEnded(): void {
if (!this.capturing && !this.finalizing) return
this.stop(); this.setStatus('Glasses disconnected. Start voice input again after reconnecting.'); this.emitEnd()
}
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
private releaseMicrophone(): void {
const session = this.session; this.session = null
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
if (!this.capturing && !this.finalizing) return
if (event.kind === 'transcript') {
this.log(`Voice: ${event.final ? 'final' : 'partial'} transcript (${event.text?.length ?? 0} characters)`)
for (const fn of [...this.transcripts]) fn({ text: event.text ?? '', isFinal: !!event.final })
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
else if (event.kind === 'audio') {
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
if (this.capturing) this.setStatus(`Listening on glasses… ${event.seconds?.toFixed(0)}s`)
} else if (event.kind === 'finishing') {
this.stopPushToTalk(); this.emitEnd()
} else if (event.kind === 'ended') {
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
}
}
}
export const voiceControlBridge = new IosVoiceControlBridge()
+3 -2
View File
@@ -116,13 +116,14 @@ export function createMainPage(): Page {
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
const footer = new GridLayout()
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
footer.columns = '*,*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
const menu = button('Menu', () => {
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
})
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
for (const [index, view] of [back, menu, keyboard].entries()) {
const voice = button('Voice', () => controller.startVoiceInput())
for (const [index, view] of [back, menu, voice, keyboard].entries()) {
view.padding = '16 8'; view.fontSize = 14
GridLayout.setColumn(view, index); footer.addChild(view)
}
+8 -3
View File
@@ -59,9 +59,14 @@ export function onAnySettingChanged(listener: () => void): () => void {
}
onSettingsStoreChanged(() => {
for (const listener of Array.from(settingChangeListeners)) {
listener();
}
// Font/cache invalidation listeners may register after this relay. Let the
// entire store notification finish before observers synchronously repaint.
// Otherwise Save updates the label but paints the previous typeface once.
setTimeout(() => {
for (const listener of Array.from(settingChangeListeners)) {
listener();
}
}, 0);
});
export abstract class ConfigSetting<TValue, TId extends string = string> {
+5 -1
View File
@@ -233,8 +233,12 @@ function settingsSections(): SettingsSection[] {
},
];
if (!global.isIOS) return sections;
const deferred = new Set(["Voice", "Assistant", "Navigate", "Watch", "Developer"]);
const deferred = new Set(["Assistant", "Navigate", "Watch", "Developer"]);
return sections.map(section => {
if (section.label === "Voice") return { label: "Voice", items: [{
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
}] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
description: `${section.label} integration has not been ported to iOS.`,
+2 -3
View File
@@ -278,7 +278,6 @@ export class FontPickerLayer implements Layer {
/** Installed fonts grouped into families (styles sorted by weight). */
function collectFamilies(monospaceOnly: boolean): FontFamily[] {
if (global.isIOS) return []; // Native TTF rendering has not been ported.
const byLabel = new Map<string, FontFamily>();
for (const font of listInstalledFonts()) {
if (monospaceOnly && !font.monospace) continue;
@@ -327,7 +326,7 @@ export function terminalFontPickerMenuItem(): MenuItem {
function fontPickerMenuItem(options: FontPickerOptions & { rowLabel: string; description: string }): MenuItem {
return {
label: options.rowLabel,
description: global.isIOS ? "Choose a bundled bitmap font. Native TTF rendering is not available on iOS yet." : options.description,
description: options.description,
onSelect: (ctx) => {
ctx.stack.push(new FontPickerLayer(options));
},
@@ -339,7 +338,7 @@ function fontPickerMenuItem(options: FontPickerOptions & { rowLabel: string; des
y,
width,
options.rowLabel,
global.isIOS && options.get().kind === "ttf" ? "Terminus (iOS)" : fontSelectionLabel(options.get()),
fontSelectionLabel(options.get()),
);
},
};
+1 -1
View File
@@ -970,7 +970,7 @@ class Shell {
*/
private buildVoiceSendTargets(): VoiceSendTarget[] {
const targets: VoiceSendTarget[] = [];
if (this.isAssistantAvailable()) {
if (!global.isIOS && this.isAssistantAvailable()) {
targets.push({
id: "assistant",
label: "Send to Assistant",
+6 -5
View File
@@ -116,9 +116,9 @@ export class VoiceInputLayer implements Layer {
this.status = state.status;
this.actions.requestRender();
});
if (this.handsFree) {
// No button is held, so the mic has to stop itself. endCapture() is
// idempotent, and a click still ends the utterance early.
if (this.handsFree || global.isIOS) {
// Endpointing or the recognizer ending its session can finish capture.
// endCapture() is idempotent, including after a manual button release.
this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => {
if (this.phase === "capturing") {
this.endCapture();
@@ -186,7 +186,7 @@ export class VoiceInputLayer implements Layer {
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
rows.push({
@@ -199,7 +199,7 @@ export class VoiceInputLayer implements Layer {
});
}
rows.push({
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
@@ -379,6 +379,7 @@ export class VoiceInputLayer implements Layer {
this.capturing = false;
void this.actions.stopVoiceCapture();
}
if (global.isIOS) voiceControlBridge.stop();
this.onClosed();
}