forked from Blendman974/faceclaw
iOS TTF support; voice input
This commit is contained in:
1 parent
ae14edcb59
commit
b07ae0b479
67 files changed
+15717
-55
No files matched your search
+14
-15
@@ -1,17 +1,16 @@
|
||||
/**
|
||||
* Font-file text rendering for the on-glasses font previewer, backed by the
|
||||
* Java FontFileRenderer (Android Typeface/Paint, so shaping, kerning, and
|
||||
* line breaking come from minikin/HarfBuzz). Rendered text arrives as
|
||||
* native font renderer (Android Paint / iOS Core Text). Rendered text arrives as
|
||||
* antialiased 8bpp coverage, which the compositor quantizes to the display's
|
||||
* 16 shades.
|
||||
*/
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { grayImageFromPacket } from "./image-files";
|
||||
|
||||
declare const com: any;
|
||||
import { fontRenderer } from "./font-renderer";
|
||||
declare const global: any;
|
||||
|
||||
/** Font file formats Android's Typeface can load. */
|
||||
/** Font file formats supported by the native renderers. */
|
||||
const FONT_FILE = /\.(ttf|otf|ttc)$/i;
|
||||
|
||||
export function isFontFile(name: string): boolean {
|
||||
@@ -37,9 +36,9 @@ export const DEFAULT_FONT_GAMMA = 1.0;
|
||||
|
||||
/** Whether the font file parses as a loadable font. */
|
||||
export function canLoadFontFile(path: string): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
if (!global.isAndroid && !global.isIOS) return false;
|
||||
try {
|
||||
return !!com.faceclaw.app.FontFileRenderer.canLoadFont(path);
|
||||
return !!fontRenderer.canLoadFont(path);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
@@ -50,9 +49,9 @@ export function canLoadFontFile(path: string): boolean {
|
||||
* unavailable (caller should fall back to the filename).
|
||||
*/
|
||||
export function fontFileDisplayName(path: string): { family: string; style: string } | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
const raw = String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? "");
|
||||
const raw = String(fontRenderer.getFontName(path) ?? "");
|
||||
if (!raw) return null;
|
||||
const [family, style] = raw.split("\n");
|
||||
if (!family) return null;
|
||||
@@ -64,9 +63,9 @@ export function fontFileDisplayName(path: string): { family: string; style: stri
|
||||
|
||||
/** Line metrics at a pixel size; null when the font cannot be loaded. */
|
||||
export function fontFileMetrics(path: string, sizePx: number): FontFileMetrics | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
const raw = String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const raw = String(fontRenderer.getFontMetrics(path, sizePx) ?? "");
|
||||
const parts = raw.split(" ").map((part) => parseInt(part, 10));
|
||||
if (parts.length !== 3 || parts.some((part) => !Number.isFinite(part))) return null;
|
||||
return { ascent: parts[0]!, descent: parts[1]!, lineGap: parts[2]! };
|
||||
@@ -86,9 +85,9 @@ export function renderFontFileText(
|
||||
sizePx: number,
|
||||
gamma = DEFAULT_FONT_GAMMA,
|
||||
): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
return grayImageFromPacket(com.faceclaw.app.FontFileRenderer.renderText(path, text, sizePx, gamma));
|
||||
return grayImageFromPacket(fontRenderer.renderText(path, text, sizePx, gamma));
|
||||
} catch (error) {
|
||||
console.warn(`renderFontFileText failed for ${path}: ${error}`);
|
||||
return null;
|
||||
@@ -96,7 +95,7 @@ export function renderFontFileText(
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a paragraph wrapped to maxWidth (StaticLayout line breaking),
|
||||
* Render a paragraph wrapped to maxWidth (native line breaking),
|
||||
* truncated with an ellipsis past maxLines.
|
||||
*/
|
||||
export function renderFontFileWrapped(opts: {
|
||||
@@ -107,10 +106,10 @@ export function renderFontFileWrapped(opts: {
|
||||
maxLines: number;
|
||||
gamma?: number;
|
||||
}): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!global.isAndroid && !global.isIOS) return null;
|
||||
try {
|
||||
return grayImageFromPacket(
|
||||
com.faceclaw.app.FontFileRenderer.renderWrapped(
|
||||
fontRenderer.renderWrapped(
|
||||
opts.path,
|
||||
opts.text,
|
||||
opts.sizePx,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
declare const FaceclawFontRenderer: any
|
||||
function bytes(data: NSData | null): Uint8Array {
|
||||
return data ? new Uint8Array(interop.bufferFromData(data)).slice() : new Uint8Array()
|
||||
}
|
||||
export const fontRenderer = {
|
||||
canLoadFont: (path: string): boolean => !!FaceclawFontRenderer.canLoadFont(path),
|
||||
getFontName: (path: string): string => String(FaceclawFontRenderer.getFontName(path) ?? ''),
|
||||
getFontMetrics: (path: string, size: number): string => String(FaceclawFontRenderer.getFontMetricsSize(path, size) ?? ''),
|
||||
measureTextExact: (path: string, text: string, size: number): number => Number(FaceclawFontRenderer.measureTextExactTextSize(path, text, size)),
|
||||
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderGlyphCellSizeCodePointGamma(path, size, cp, gamma)),
|
||||
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => bytes(FaceclawFontRenderer.renderTextTextSizeGamma(path, text, size, gamma)),
|
||||
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => bytes(FaceclawFontRenderer.renderWrappedTextSizeWidthGammaMaxLines(path, text, size, width, gamma, lines)),
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/** Platform boundary for font rasterization; layout and glyph caches stay shared. */
|
||||
import { toUint8Array } from '../util/array-util'
|
||||
declare const com: any
|
||||
export const fontRenderer = {
|
||||
canLoadFont: (path: string): boolean => !!com.faceclaw.app.FontFileRenderer.canLoadFont(path),
|
||||
getFontName: (path: string): string => String(com.faceclaw.app.FontFileRenderer.getFontName(path) ?? ''),
|
||||
getFontMetrics: (path: string, size: number): string => String(com.faceclaw.app.FontFileRenderer.getFontMetrics(path, size) ?? ''),
|
||||
measureTextExact: (path: string, text: string, size: number): number => Number(com.faceclaw.app.FontFileRenderer.measureTextExact(path, text, size)),
|
||||
renderGlyphCell: (path: string, size: number, cp: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderGlyphCell(path, size, cp, gamma)),
|
||||
renderText: (path: string, text: string, size: number, gamma: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderText(path, text, size, gamma)),
|
||||
renderWrapped: (path: string, text: string, size: number, width: number, gamma: number, lines: number): Uint8Array => toUint8Array(com.faceclaw.app.FontFileRenderer.renderWrapped(path, text, size, width, gamma, lines)),
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { voiceControlBridge as iosVoiceInput } from './voice-control.ios'
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
|
||||
}
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* exclusively from the connected glasses, never from the phone microphone.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
private readonly statuses = new Set<(state: VoiceControlState) => void>()
|
||||
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
|
||||
private readonly ends = new Set<() => void>()
|
||||
private status = 'Voice input ready.'
|
||||
private generation = 0
|
||||
private session: IosMicrophoneSession | null = null
|
||||
private capturing = false
|
||||
private finalizing = false
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status }); return () => { this.statuses.delete(listener) }
|
||||
}
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcripts.add(listener); return () => { this.transcripts.delete(listener) }
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
}
|
||||
return this.native
|
||||
}
|
||||
async prepare(foreground: boolean): Promise<boolean> {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.session = session; this.packets = 0; this.capturing = true
|
||||
const error = String(this.ensureNative().start() ?? '')
|
||||
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
this.packets++
|
||||
const copy = new Uint8Array(packet)
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log('Voice: glasses microphone enabled; using on-device recognition')
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
this.stop(); this.setStatus('No microphone audio received. Reconnect the glasses and try again.'); this.emitEnd()
|
||||
}
|
||||
}, 5000)
|
||||
} catch (error) {
|
||||
if (generation !== this.generation) return
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
stopPushToTalk(): void {
|
||||
if (!this.capturing) return
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus('Recognizing…'); this.native?.finish()
|
||||
}
|
||||
stop(): void {
|
||||
++this.generation
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
}
|
||||
handleSessionEnded(): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
this.stop(); this.setStatus('Glasses disconnected. Start voice input again after reconnecting.'); this.emitEnd()
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
const session = this.session; this.session = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
if (event.kind === 'transcript') {
|
||||
this.log(`Voice: ${event.final ? 'final' : 'partial'} transcript (${event.text?.length ?? 0} characters)`)
|
||||
for (const fn of [...this.transcripts]) fn({ text: event.text ?? '', isFinal: !!event.final })
|
||||
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
|
||||
else if (event.kind === 'audio') {
|
||||
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
|
||||
if (this.capturing) this.setStatus(`Listening on glasses… ${event.seconds?.toFixed(0)}s`)
|
||||
} else if (event.kind === 'finishing') {
|
||||
this.stopPushToTalk(); this.emitEnd()
|
||||
} else if (event.kind === 'ended') {
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
}
|
||||
}
|
||||
export const voiceControlBridge = new IosVoiceControlBridge()
|
||||
Reference in new issue
Block a user