forked from Blendman974/faceclaw
iOS TTF support; voice input
This commit is contained in:
1 parent
ae14edcb59
commit
b07ae0b479
67 files changed
+15717
-55
No files matched your search
@@ -45,7 +45,7 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
|
||||
let created: InProcessWindow | null = null;
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) => TEXT_FILE.test(name) || (!global.isIOS && (isDecodableImageFile(name) || EHPK_FILE.test(name) || isFontFile(name))),
|
||||
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || (!global.isIOS && (isDecodableImageFile(name) || EHPK_FILE.test(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -170,7 +170,7 @@ function confirmEhpkPermissions(ctx: LayerContext, entry: DirectoryEntry, procee
|
||||
* window for viewable types, empty for everything else (metadata only).
|
||||
*/
|
||||
function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileInfoAction[] {
|
||||
if (global.isIOS && !TEXT_FILE.test(entry.name)) return [];
|
||||
if (global.isIOS && !TEXT_FILE.test(entry.name) && !isFontFile(entry.name)) return [];
|
||||
if (TEXT_FILE.test(entry.name)) {
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -27,7 +27,7 @@ const PARAGRAPH =
|
||||
|
||||
/**
|
||||
* Font previewer for a font file on disk (TTF/OTF/TTC), rendered through the
|
||||
* Android text stack: a single-line pangram, a character-set sample, and a
|
||||
* native text stack: a single-line pangram, a character-set sample, and a
|
||||
* wrapped paragraph, all at an adjustable pixel size. Scroll changes size,
|
||||
* click cycles the coverage gamma (for judging antialiasing on the lens),
|
||||
* double-click closes. Sized to its hosting stack (a Files document window,
|
||||
|
||||
@@ -21,6 +21,7 @@ const wrap = (command: number, magic: number, field: number, body: Uint8Array) =
|
||||
export const authentication = (magic: number) => wrap(4, magic, 3, concat(integer(1, 1), integer(2, 4)))
|
||||
export const prelude = () => wrap(2, 156, 4, bytes(3, bytes(2, bytes(2, concat(integer(1, 0), integer(2, 0))))))
|
||||
export const heartbeat = (magic: number) => wrap(12, magic, 14, integer(1, 0))
|
||||
export const audioControl = (magic: number, enabled: boolean) => wrap(15, magic, 18, integer(1, enabled ? 1 : 0))
|
||||
export const settingsQuery = (magic: number) => wrap(2, magic, 4, integer(1, 1))
|
||||
export const shutdown = (magic: number) => wrap(9, magic, 11, integer(1, 1))
|
||||
export const framebufferLease = (acquire: boolean) => concat(integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, acquire ? 5 : 6, 0, 0])))
|
||||
|
||||
@@ -51,6 +51,10 @@ export class GlassesSession {
|
||||
private layoutCreated = false
|
||||
private off: () => void
|
||||
private stopping: Promise<void> | null = null
|
||||
private audioListener: ((packet: Uint8Array) => void) | null = null
|
||||
private microphoneToken = 0
|
||||
private microphoneWork: Promise<void> = Promise.resolve()
|
||||
private microphoneEnabled = false
|
||||
constructor(private readonly transport: SessionTransport,
|
||||
private readonly deflate: (data: Uint8Array) => Uint8Array,
|
||||
private readonly onState: (state: SessionState) => void,
|
||||
@@ -211,6 +215,13 @@ export class GlassesSession {
|
||||
return
|
||||
}
|
||||
if (event.kind !== 'notification' || !event.data) return
|
||||
if (event.characteristic === protocol.G2_RENDER_NOTIFY && event.identifier !== this.ids.ring) {
|
||||
if (this.audioListener && this.state.phase === 'connected') {
|
||||
this.audioListener(hexToBytes(event.data))
|
||||
this.onActivity(); this.wake()
|
||||
}
|
||||
return
|
||||
}
|
||||
const ring = event.identifier === this.ids.ring
|
||||
if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return
|
||||
try {
|
||||
@@ -268,6 +279,30 @@ export class GlassesSession {
|
||||
if (this.state.phase !== 'connected') return
|
||||
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
|
||||
}
|
||||
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
|
||||
const token = ++this.microphoneToken, generation = this.generation
|
||||
this.audioListener = enabled ? listener ?? null : null
|
||||
const current = () => generation === this.generation && token === this.microphoneToken
|
||||
const work = this.microphoneWork.catch(() => {}).then(async () => {
|
||||
if (!current()) return
|
||||
if (this.state.phase !== 'connected') {
|
||||
if (enabled) throw new Error('Connect the glasses before starting voice input.')
|
||||
return
|
||||
}
|
||||
if (enabled) {
|
||||
await Promise.all(['left', 'right'].map(role => this.transport.subscribe(this.ids[role], protocol.G2_RENDER_NOTIFY)))
|
||||
if (!current()) return
|
||||
}
|
||||
await this.request('right', protocol.SID.hub, magic => protocol.audioControl(magic, enabled), enabled ? 'Microphone enable' : 'Microphone disable')
|
||||
if (current()) { this.microphoneEnabled = enabled; this.log(`Glasses microphone ${enabled ? 'enabled' : 'disabled'}`) }
|
||||
}).catch(error => {
|
||||
if (current()) { this.audioListener = null; this.microphoneEnabled = false }
|
||||
throw error
|
||||
})
|
||||
this.microphoneWork = work
|
||||
return work
|
||||
}
|
||||
wake(): void {
|
||||
if (this.timer !== null) { clearTimeout(this.timer); this.timer = null }
|
||||
void this.pump()
|
||||
@@ -308,6 +343,7 @@ export class GlassesSession {
|
||||
}
|
||||
private reset(): void {
|
||||
++this.generation
|
||||
++this.microphoneToken; this.audioListener = null; this.microphoneEnabled = false
|
||||
if (this.timer !== null) clearTimeout(this.timer)
|
||||
if (this.retryTimer !== null) clearTimeout(this.retryTimer)
|
||||
if (this.ringRetryTimer !== null) clearTimeout(this.ringRetryTimer)
|
||||
@@ -337,6 +373,11 @@ export class GlassesSession {
|
||||
}
|
||||
private async stopInternal(): Promise<void> {
|
||||
this.wanted = false
|
||||
// Stop mic capture before ending the display session. Late enable ACKs
|
||||
// cannot leave a microphone streaming after an explicit Disconnect.
|
||||
if (this.audioListener || this.microphoneEnabled) {
|
||||
try { await this.setMicrophone(false) } catch (error) { this.log(`Microphone cleanup: ${this.message(error)}`) }
|
||||
}
|
||||
const cleanup = this.layoutCreated && this.state.phase === 'connected'
|
||||
this.update('disconnecting', 'Disconnecting…'); this.reset()
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescript/core'
|
||||
import { iosBluetooth } from '../native/ios-bluetooth'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { GlassesSession, type SessionState } from './glasses-session'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
@@ -47,7 +48,8 @@ export class IosPreviewController {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: () => this.typeIntoApp(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startTextSettingEdit: setting => this.editSetting(setting),
|
||||
startTextSettingsEdit: async (settings, _title, finished) => {
|
||||
@@ -62,7 +64,8 @@ export class IosPreviewController {
|
||||
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
voiceInputEnabled: false,
|
||||
voiceInputEnabled: true,
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => this.requestShellRender(),
|
||||
@@ -285,7 +288,7 @@ export class IosPreviewController {
|
||||
openSettings: section => { void this.launchApp('settings', { section }) },
|
||||
startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) },
|
||||
endTextSettingEdit: () => {},
|
||||
startTextInput: () => { void this.typeIntoApp() },
|
||||
startTextInput: () => shell.startVoiceInput(),
|
||||
})
|
||||
this.appHosts.set(appId, host)
|
||||
return host
|
||||
@@ -307,6 +310,7 @@ export class IosPreviewController {
|
||||
if (!this.session) {
|
||||
const { deflate } = require('pako') as { deflate: (data: Uint8Array) => Uint8Array }
|
||||
this.session = new GlassesSession(iosBluetooth(), deflate, state => {
|
||||
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState(state)
|
||||
@@ -322,6 +326,20 @@ export class IosPreviewController {
|
||||
await this.session.start(addresses)
|
||||
}
|
||||
async disconnect(): Promise<void> { await this.session?.stop() }
|
||||
startVoiceInput(): void { shell.startVoiceInput() }
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (this.session?.state.phase !== 'connected') {
|
||||
if (this.active) this.onError('Connect the glasses to use their microphone.')
|
||||
return false
|
||||
}
|
||||
const ready = await iosVoiceInput.prepare(this.active)
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && this.session?.state.phase === 'connected'
|
||||
}
|
||||
private async startVoiceCapture(): Promise<void> {
|
||||
if (!this.session || this.session.state.phase !== 'connected') return
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
||||
|
||||
@@ -4,14 +4,14 @@
|
||||
* app assets on first listing (so the directory is the single source of
|
||||
* truth), and user installs from the Files app ("Install" on a font file).
|
||||
*
|
||||
* Listing parses each file's name table for family/style (via the Java
|
||||
* FontFileRenderer) and probes monospace-ness by comparing advances, both
|
||||
* Listing parses each file's name table for family/style (via the native
|
||||
* font renderer) and probes monospace-ness by comparing advances, both
|
||||
* memoized per path for the session.
|
||||
*/
|
||||
import { File, Folder, knownFolders, path as nsPath } from "@nativescript/core";
|
||||
import { canLoadFontFile, fontFileDisplayName, isFontFile } from "../native/font-files";
|
||||
|
||||
declare const com: any;
|
||||
import { fontRenderer } from "../native/font-renderer";
|
||||
declare const java: any;
|
||||
declare const global: any;
|
||||
|
||||
@@ -62,7 +62,7 @@ let preinstalledEnsured = false;
|
||||
* before every listing.
|
||||
*/
|
||||
export function ensurePreinstalledFonts(): void {
|
||||
if (preinstalledEnsured || !global.isAndroid) return;
|
||||
if (preinstalledEnsured || (!global.isAndroid && !global.isIOS)) return;
|
||||
const assetsDir = nsPath.join(knownFolders.currentApp().path, "fonts/ttf");
|
||||
for (const fileName of PREINSTALLED_FONT_FILES) {
|
||||
const target = installedFontPath(fileName);
|
||||
@@ -83,7 +83,7 @@ export function ensurePreinstalledFonts(): void {
|
||||
* same-named file). Returns an error message for the UI, or null on success.
|
||||
*/
|
||||
export function installFontFile(sourcePath: string): string | null {
|
||||
if (!global.isAndroid) return "Not available on this platform.";
|
||||
if ((!global.isAndroid && !global.isIOS)) return "Not available on this platform.";
|
||||
if (!isFontFile(sourcePath)) return "Not a font file.";
|
||||
if (!canLoadFontFile(sourcePath)) return "This file could not be loaded as a font.";
|
||||
const fileName = sourcePath.slice(sourcePath.lastIndexOf("/") + 1);
|
||||
@@ -108,7 +108,7 @@ export function isFontInstalled(sourcePath: string): boolean {
|
||||
* then weight. Unloadable files are skipped.
|
||||
*/
|
||||
export function listInstalledFonts(): InstalledFont[] {
|
||||
if (!global.isAndroid) return [];
|
||||
if ((!global.isAndroid && !global.isIOS)) return [];
|
||||
ensurePreinstalledFonts();
|
||||
const fonts: InstalledFont[] = [];
|
||||
try {
|
||||
@@ -195,7 +195,7 @@ function styleWeightOrder(style: string): number {
|
||||
function isMonospaceFont(path: string): boolean {
|
||||
try {
|
||||
const measure = (text: string) =>
|
||||
Number(com.faceclaw.app.FontFileRenderer.measureTextExact(path, text, 48));
|
||||
Number(fontRenderer.measureTextExact(path, text, 48));
|
||||
const narrow = measure("i");
|
||||
if (!(narrow > 0)) return false;
|
||||
return (
|
||||
@@ -217,6 +217,14 @@ function isMonospaceFont(path: string): boolean {
|
||||
// isolates on first launch (each resolving the default UI font), and readers
|
||||
// must never see a half-written font file.
|
||||
function copyFileBytes(source: string, target: string): void {
|
||||
if (source === target) return;
|
||||
if (global.isIOS) {
|
||||
const data = NSData.dataWithContentsOfFile(source);
|
||||
// Foundation's atomic write uses a sibling temporary file and rename.
|
||||
// Multiple worker isolates can safely preinstall the same bundled face.
|
||||
if (!data || !data.writeToFileAtomically(target, true)) throw new Error(`Copy to ${target} failed`);
|
||||
return;
|
||||
}
|
||||
const temp = `${target}.tmp-${Math.floor(Math.random() * 0x7fffffff).toString(36)}`;
|
||||
const input = new java.io.FileInputStream(source);
|
||||
try {
|
||||
|
||||
+10
-11
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* TTF/OTF fonts for on-glasses UI text, rendered per-codepoint through the
|
||||
* Java FontFileRenderer (Android minikin/HarfBuzz: real shaping, kerning,
|
||||
* antialiasing) and cached as 4bpp coverage rasters that ride the
|
||||
* native font renderer (Android Paint / iOS Core Text) and cached as 4bpp coverage rasters that ride the
|
||||
* texture-cache glyph pipeline (CFW mode 14; see
|
||||
* notes/texture-cache-display-list-design.md and graphics/glyph-wire.ts).
|
||||
*
|
||||
@@ -14,9 +13,9 @@
|
||||
* never drifts.
|
||||
*
|
||||
* Kerning uses memoized per-pair measurements (measure("ab") - a - b);
|
||||
* ligatures are disabled Java-side so per-codepoint rasters plus pairwise
|
||||
* ligatures are disabled native-side so per-codepoint rasters plus pairwise
|
||||
* kerning fully describe a line. Glyph rasters and advances come from one
|
||||
* JNI render per (font, size, codepoint) per JS context, memoized forever.
|
||||
* native render per (font, size, codepoint) per JS context, memoized forever.
|
||||
*
|
||||
* Cross-context determinism: worker threads and the main thread each build
|
||||
* their own TtfFont, but the Java GlyphAtlas dedupes by atlasKey and treats
|
||||
@@ -26,8 +25,8 @@
|
||||
import { allocateFontFingerprintId, type Glyph } from "./bdffont";
|
||||
import { GrayImage, grayToNibble, type GlyphFont } from "./image";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { fontRenderer } from "../native/font-renderer";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
|
||||
/**
|
||||
@@ -65,7 +64,7 @@ export class TtfFont implements GlyphFont {
|
||||
|
||||
/**
|
||||
* Load (or return the memoized) font at a pixel size. Null when the font
|
||||
* file is missing/unloadable or off-Android — callers fall back to a BDF
|
||||
* file is missing/unloadable or the platform has no renderer — callers fall back to a BDF
|
||||
* face.
|
||||
*/
|
||||
static load(path: string, sizePx: number, gamma = DEFAULT_TTF_GAMMA): TtfFont | null {
|
||||
@@ -73,9 +72,9 @@ export class TtfFont implements GlyphFont {
|
||||
const cached = loadedFonts.get(key);
|
||||
if (cached !== undefined) return cached;
|
||||
let font: TtfFont | null = null;
|
||||
if (global.isAndroid && sizePx > 0) {
|
||||
if ((global.isAndroid || global.isIOS) && Number.isFinite(sizePx) && sizePx > 0) {
|
||||
try {
|
||||
const raw = String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const raw = String(fontRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const parts = raw.split(" ").map((part) => parseInt(part, 10));
|
||||
if (parts.length === 3 && parts.every((part) => Number.isFinite(part))) {
|
||||
const [ascent, descent] = parts as [number, number, number];
|
||||
@@ -170,7 +169,7 @@ export class TtfFont implements GlyphFont {
|
||||
|
||||
/**
|
||||
* Kerning between two adjacent codepoints: measure(ab) - a - b, memoized.
|
||||
* One JNI measurement per distinct pair per JS context.
|
||||
* One native measurement per distinct pair per JS context.
|
||||
*/
|
||||
private kernBetween(a: number, b: number): number {
|
||||
const key = a * 0x200000 + b;
|
||||
@@ -179,7 +178,7 @@ export class TtfFont implements GlyphFont {
|
||||
let kern = 0;
|
||||
try {
|
||||
const pair = Number(
|
||||
com.faceclaw.app.FontFileRenderer.measureTextExact(
|
||||
fontRenderer.measureTextExact(
|
||||
this.path,
|
||||
String.fromCodePoint(a) + String.fromCodePoint(b),
|
||||
this.sizePx,
|
||||
@@ -203,7 +202,7 @@ export class TtfFont implements GlyphFont {
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = toUint8Array(
|
||||
com.faceclaw.app.FontFileRenderer.renderGlyphCell(this.path, this.sizePx, codePoint, this.gamma),
|
||||
fontRenderer.renderGlyphCell(this.path, this.sizePx, codePoint, this.gamma),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`renderGlyphCell failed for U+${codePoint.toString(16)}: ${error}`);
|
||||
|
||||
+14
-15
@@ -1,17 +1,16 @@
|
||||
/**
|
||||
* Font-file text rendering for the on-glasses font previewer, backed by the
|
||||
* Java FontFileRenderer (Android Typeface/Paint, so shaping, kerning, and
|
||||
* line breaking come from minikin/HarfBuzz). Rendered text arrives as
|
||||
* native font renderer (Android Paint / iOS Core Text). Rendered text arrives as
|
||||
* antialiased 8bpp coverage, which the compositor quantizes to the display's
|
||||
* 16 shades.
|
||||
*/
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { grayImageFromPacket } from "./image-files";
|
||||
|
||||
declare const com: any;
|
||||
import { fontRenderer } from "./font-renderer";
|
||||
declare const global: any;
|
||||
|
||||
/** Font file formats Android's Typeface can load. */
|
||||
/** Font file formats supported by the native renderers. */
|
||||
const FONT_FILE = /\.(ttf|otf|ttc)$/i;
|
||||
|
||||
export function isFontFile(name: string): boolean {
|
||||
@@ -37,9 +36,9 @@ export const DEFAULT_FONT_GAMMA = 1.0;
|
||||
|
||||
/** Whether the font file parses as a loadable font. */
|
||||
export function canLoadFontFile(path: string): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
if (!global.isAndroid && !global.isIOS) return false;
|
||||
try {
|
||||
return !!com.faceclaw.app.FontFileRenderer.canLoadFont(path);
|
||||
return !!fontRenderer.canLoadFont(path);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
@@ -50,9 +49,9 @@ export function canLoadFontFile(path: string): boolean {
|
||||
* unavailable (caller should fall back to the filename).
|
||||
*/
|
||||
export function fontFileDisplayName(path: string): { family: string; style: string } | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
const raw = String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? "");
|
||||
const raw = String(fontRenderer.getFontName(path) ?? "");
|
||||
if (!raw) return null;
|
||||
const [family, style] = raw.split("\n");
|
||||
if (!family) return null;
|
||||
@@ -64,9 +63,9 @@ export function fontFileDisplayName(path: string): { family: string; style: stri
|
||||
|
||||
/** Line metrics at a pixel size; null when the font cannot be loaded. */
|
||||
export function fontFileMetrics(path: string, sizePx: number): FontFileMetrics | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
const raw = String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const raw = String(fontRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const parts = raw.split(" ").map((part) => parseInt(part, 10));
|
||||
if (parts.length !== 3 || parts.some((part) => !Number.isFinite(part))) return null;
|
||||
return { ascent: parts[0]!, descent: parts[1]!, lineGap: parts[2]! };
|
||||
@@ -86,9 +85,9 @@ export function renderFontFileText(
|
||||
sizePx: number,
|
||||
gamma = DEFAULT_FONT_GAMMA,
|
||||
): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
return grayImageFromPacket(com.faceclaw.app.FontFileRenderer.renderText(path, text, sizePx, gamma));
|
||||
return grayImageFromPacket(fontRenderer.renderText(path, text, sizePx, gamma));
|
||||
} catch (error) {
|
||||
console.warn(`renderFontFileText failed for ${path}: ${error}`);
|
||||
return null;
|
||||
@@ -96,7 +95,7 @@ export function renderFontFileText(
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a paragraph wrapped to maxWidth (StaticLayout line breaking),
|
||||
* Render a paragraph wrapped to maxWidth (native line breaking),
|
||||
* truncated with an ellipsis past maxLines.
|
||||
*/
|
||||
export function renderFontFileWrapped(opts: {
|
||||
@@ -107,10 +106,10 @@ export function renderFontFileWrapped(opts: {
|
||||
maxLines: number;
|
||||
gamma?: number;
|
||||
}): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
return grayImageFromPacket(
|
||||
com.faceclaw.app.FontFileRenderer.renderWrapped(
|
||||
fontRenderer.renderWrapped(
|
||||
opts.path,
|
||||
opts.text,
|
||||
opts.sizePx,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
declare const FaceclawFontRenderer: any
|
||||
function bytes(data: NSData | null): Uint8Array {
|
||||
return data ? new Uint8Array(interop.bufferFromData(data)).slice() : new Uint8Array()
|
||||
}
|
||||
export const fontRenderer = {
|
||||
canLoadFont: (path: string): boolean => !!FaceclawFontRenderer.canLoadFont(path),
|
||||
getFontName: (path: string): string => String(FaceclawFontRenderer.getFontName(path) ?? ''),
|
||||
getFontMetrics: (path: string, size: number): string => String(FaceclawFontRenderer.getFontMetricsSize(path, size) ?? ''),
|
||||
measureTextExact: (path: string, text: string, size: number): number => Number(FaceclawFontRenderer.measureTextExactTextSize(path, text, size)),
|
||||
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderGlyphCellSizeCodePointGamma(path, size, cp, gamma)),
|
||||
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderTextTextSizeGamma(path, text, size, gamma)),
|
||||
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => bytes(FaceclawFontRenderer.renderWrappedTextSizeWidthGammaMaxLines(path, text, size, width, gamma, lines)),
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/** Platform boundary for font rasterization; layout and glyph caches stay shared. */
|
||||
import { toUint8Array } from '../util/array-util'
|
||||
declare const com: any
|
||||
export const fontRenderer = {
|
||||
canLoadFont: (path: string): boolean => !!com.faceclaw.app.FontFileRenderer.canLoadFont(path),
|
||||
getFontName: (path: string): string => String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? ''),
|
||||
getFontMetrics: (path: string, size: number): string => String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, size) ?? ''),
|
||||
measureTextExact: (path: string, text: string, size: number): number => Number(com.faceclaw.app.FontFileRenderer.measureTextExact(path, text, size)),
|
||||
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderGlyphCell(path, size, cp, gamma)),
|
||||
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderText(path, text, size, gamma)),
|
||||
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderWrapped(path, text, size, width, gamma, lines)),
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { voiceControlBridge as iosVoiceInput } from './voice-control.ios'
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
|
||||
}
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* exclusively from the connected glasses, never from the phone microphone.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
private readonly statuses = new Set<(state: VoiceControlState) => void>()
|
||||
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
|
||||
private readonly ends = new Set<() => void>()
|
||||
private status = 'Voice input ready.'
|
||||
private generation = 0
|
||||
private session: IosMicrophoneSession | null = null
|
||||
private capturing = false
|
||||
private finalizing = false
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status }); return () => { this.statuses.delete(listener) }
|
||||
}
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcripts.add(listener); return () => { this.transcripts.delete(listener) }
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
}
|
||||
return this.native
|
||||
}
|
||||
async prepare(foreground: boolean): Promise<boolean> {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.session = session; this.packets = 0; this.capturing = true
|
||||
const error = String(this.ensureNative().start() ?? '')
|
||||
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
this.packets++
|
||||
const copy = new Uint8Array(packet)
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log('Voice: glasses microphone enabled; using on-device recognition')
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
this.stop(); this.setStatus('No microphone audio received. Reconnect the glasses and try again.'); this.emitEnd()
|
||||
}
|
||||
}, 5000)
|
||||
} catch (error) {
|
||||
if (generation !== this.generation) return
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
stopPushToTalk(): void {
|
||||
if (!this.capturing) return
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus('Recognizing…'); this.native?.finish()
|
||||
}
|
||||
stop(): void {
|
||||
++this.generation
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
}
|
||||
handleSessionEnded(): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
this.stop(); this.setStatus('Glasses disconnected. Start voice input again after reconnecting.'); this.emitEnd()
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
const session = this.session; this.session = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
if (event.kind === 'transcript') {
|
||||
this.log(`Voice: ${event.final ? 'final' : 'partial'} transcript (${event.text?.length ?? 0} characters)`)
|
||||
for (const fn of [...this.transcripts]) fn({ text: event.text ?? '', isFinal: !!event.final })
|
||||
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
|
||||
else if (event.kind === 'audio') {
|
||||
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
|
||||
if (this.capturing) this.setStatus(`Listening on glasses… ${event.seconds?.toFixed(0)}s`)
|
||||
} else if (event.kind === 'finishing') {
|
||||
this.stopPushToTalk(); this.emitEnd()
|
||||
} else if (event.kind === 'ended') {
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
}
|
||||
}
|
||||
export const voiceControlBridge = new IosVoiceControlBridge()
|
||||
@@ -116,13 +116,14 @@ export function createMainPage(): Page {
|
||||
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
|
||||
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
|
||||
const footer = new GridLayout()
|
||||
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
|
||||
footer.columns = '*,*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
|
||||
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
|
||||
const menu = button('Menu', () => {
|
||||
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
|
||||
})
|
||||
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
|
||||
for (const [index, view] of [back, menu, keyboard].entries()) {
|
||||
const voice = button('Voice', () => controller.startVoiceInput())
|
||||
for (const [index, view] of [back, menu, voice, keyboard].entries()) {
|
||||
view.padding = '16 8'; view.fontSize = 14
|
||||
GridLayout.setColumn(view, index); footer.addChild(view)
|
||||
}
|
||||
|
||||
@@ -59,9 +59,14 @@ export function onAnySettingChanged(listener: () => void): () => void {
|
||||
}
|
||||
|
||||
onSettingsStoreChanged(() => {
|
||||
for (const listener of Array.from(settingChangeListeners)) {
|
||||
listener();
|
||||
}
|
||||
// Font/cache invalidation listeners may register after this relay. Let the
|
||||
// entire store notification finish before observers synchronously repaint.
|
||||
// Otherwise Save updates the label but paints the previous typeface once.
|
||||
setTimeout(() => {
|
||||
for (const listener of Array.from(settingChangeListeners)) {
|
||||
listener();
|
||||
}
|
||||
}, 0);
|
||||
});
|
||||
|
||||
export abstract class ConfigSetting<TValue, TId extends string = string> {
|
||||
|
||||
@@ -233,8 +233,12 @@ function settingsSections(): SettingsSection[] {
|
||||
},
|
||||
];
|
||||
if (!global.isIOS) return sections;
|
||||
const deferred = new Set(["Voice", "Assistant", "Navigate", "Watch", "Developer"]);
|
||||
const deferred = new Set(["Assistant", "Navigate", "Watch", "Developer"]);
|
||||
return sections.map(section => {
|
||||
if (section.label === "Voice") return { label: "Voice", items: [{
|
||||
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
|
||||
}] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
|
||||
@@ -278,7 +278,6 @@ export class FontPickerLayer implements Layer {
|
||||
|
||||
/** Installed fonts grouped into families (styles sorted by weight). */
|
||||
function collectFamilies(monospaceOnly: boolean): FontFamily[] {
|
||||
if (global.isIOS) return []; // Native TTF rendering has not been ported.
|
||||
const byLabel = new Map<string, FontFamily>();
|
||||
for (const font of listInstalledFonts()) {
|
||||
if (monospaceOnly && !font.monospace) continue;
|
||||
@@ -327,7 +326,7 @@ export function terminalFontPickerMenuItem(): MenuItem {
|
||||
function fontPickerMenuItem(options: FontPickerOptions & { rowLabel: string; description: string }): MenuItem {
|
||||
return {
|
||||
label: options.rowLabel,
|
||||
description: global.isIOS ? "Choose a bundled bitmap font. Native TTF rendering is not available on iOS yet." : options.description,
|
||||
description: options.description,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new FontPickerLayer(options));
|
||||
},
|
||||
@@ -339,7 +338,7 @@ function fontPickerMenuItem(options: FontPickerOptions & { rowLabel: string; des
|
||||
y,
|
||||
width,
|
||||
options.rowLabel,
|
||||
global.isIOS && options.get().kind === "ttf" ? "Terminus (iOS)" : fontSelectionLabel(options.get()),
|
||||
fontSelectionLabel(options.get()),
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -970,7 +970,7 @@ class Shell {
|
||||
*/
|
||||
private buildVoiceSendTargets(): VoiceSendTarget[] {
|
||||
const targets: VoiceSendTarget[] = [];
|
||||
if (this.isAssistantAvailable()) {
|
||||
if (!global.isIOS && this.isAssistantAvailable()) {
|
||||
targets.push({
|
||||
id: "assistant",
|
||||
label: "Send to Assistant",
|
||||
|
||||
@@ -116,9 +116,9 @@ export class VoiceInputLayer implements Layer {
|
||||
this.status = state.status;
|
||||
this.actions.requestRender();
|
||||
});
|
||||
if (this.handsFree) {
|
||||
// No button is held, so the mic has to stop itself. endCapture() is
|
||||
// idempotent, and a click still ends the utterance early.
|
||||
if (this.handsFree || global.isIOS) {
|
||||
// Endpointing or the recognizer ending its session can finish capture.
|
||||
// endCapture() is idempotent, including after a manual button release.
|
||||
this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => {
|
||||
if (this.phase === "capturing") {
|
||||
this.endCapture();
|
||||
@@ -186,7 +186,7 @@ export class VoiceInputLayer implements Layer {
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
rows.push({
|
||||
@@ -199,7 +199,7 @@ export class VoiceInputLayer implements Layer {
|
||||
});
|
||||
}
|
||||
rows.push({
|
||||
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
@@ -379,6 +379,7 @@ export class VoiceInputLayer implements Layer {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
}
|
||||
if (global.isIOS) voiceControlBridge.stop();
|
||||
this.onClosed();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user