iOS porting: Main screen and compositor

This commit is contained in:
James Babcock committed 2026-09-05 22:59:37 -07:00
1 parent 61b3bf8b7c
commit 99793a37a7
24 files changed
+1124 -70

No files matched your search

+8 -4
View File
@@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer {
constructor(
private readonly coordinator: MathCoordinator,
private readonly actions: LayerActions,
private readonly textInput?: () => Promise<void>,
) {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
}
private get mode(): CalculatorMode {
@@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer {
}
private refreshListening(): void {
if (this.textInput) return;
if (!this.foreground) return;
if (this.listening === "continuous") {
if (!this.continuousActive) {
@@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer {
* that did not.
*/
beginDictation(): void {
if (this.textInput) { void this.textInput(); return; }
if (this.dictating) return;
this.dictating = true;
void this.actions.startVoiceCapture(true);
@@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer {
bodyBottom: number,
): void {
const prompt =
this.listening === "continuous"
this.textInput ? "Tap to type a problem on your phone"
: this.listening === "continuous"
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
: "Tap to say a problem — or use Voice input to type one";
const lines = wrapText(font, prompt, width - 96);
@@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer {
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
if (!this.lastInput) return;
const label = "Heard";
const label = this.textInput ? "Entered" : "Heard";
image.drawText(font, 24, y, label, 110);
const x = 24 + font.measureText(label) + 10;
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
@@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer {
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
return gestureHints([
[GESTURE_SCROLL, scrollTarget],
[GESTURE_CLICK, "mic"],
[GESTURE_CLICK, this.textInput ? "type" : "mic"],
[GESTURE_DOUBLE_CLICK, "back"],
]);
}
+3 -46
View File
@@ -1,47 +1,4 @@
import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core'
import { bytesToHex, hexToBytes } from './util/hex-util'
import { Application } from '@nativescript/core'
import { createMainPage } from './phone-ui/main-page.ios'
// Milestone 1: exercise native UI, TypeScript, and an existing shared utility
// without loading the Android services used by the full glasses application.
Application.run({
create: () => {
const page = new Page()
page.actionBarHidden = true
const scroll = new ScrollView()
const content = new StackLayout()
content.padding = 28
content.verticalAlignment = 'middle'
const label = (text: string, fontSize = 17) => {
const view = new Label()
view.text = text
view.fontSize = fontSize
view.textWrap = true
view.marginBottom = 20
content.addChild(view)
return view
}
label('Faceclaw', 36)
label('iOS port · First launch', 24)
label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`)
label('Glasses pairing and the Android features are still to be ported.')
const status = label('Tap below to test native input and shared TypeScript code.')
const button = new Button()
button.text = 'Run shared-code check'
button.accessibilityIdentifier = 'shared-code-check'
let checks = 0
button.on('tap', () => {
const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77'))
const passed = result === '46616365636C6177'
checks += 1
status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.`
console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`)
})
content.addChild(button)
scroll.content = content
page.content = scroll
page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded'))
return page
},
})
Application.run({ create: createMainPage })
+220
View File
@@ -0,0 +1,220 @@
import { Dialogs, type ImageSource } from '@nativescript/core'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
import { CalculatorLayer } from '../apps/calculator/calculator'
import { MathCoordinator } from '../apps/calculator/math/coordinator'
import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
import { noopLayerActions, type LayerActions } from '../ui/layers'
import { MenuLayer } from '../ui/menu'
import { createInProcessWindow } from '../ui/shell/in-process-window'
import { shell, type ShellWindow } from '../ui/shell/shell'
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
/** Local display host: shared shell/windows, no device-service initialization. */
export class IosPreviewController {
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
private renderTimer: ReturnType<typeof setTimeout> | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null
private active = false
private shellDirty = true
private inputQueue: Promise<void> = Promise.resolve()
private lastLayout = ''
private prompting = false
private readonly actions: LayerActions = {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
startTextSettingEdit: setting => this.editSetting(setting),
startTextSettingsEdit: async (settings, _title, finished) => {
for (const setting of settings) await this.editSetting(setting)
finished?.()
},
}
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
private readonly onError: (message: string) => void) {
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
shell.configure({
actions: this.actions,
voiceInputEnabled: false,
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
})
const launcher = createLauncherWindow({
actions: this.actions,
apps: () => [
{ appId: 'calculator', label: 'Calculator', icon: 'calculator' },
{ appId: 'display', label: 'Display', icon: 'settings' },
],
launchApp: id => this.launchApp(id),
uninstallApp: () => {},
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() },
})
this.configureWindow(launcher)
shell.registerWindow(launcher)
shell.wake('window')
shell.focusWindow(launcher.windowId)
}
resume(): void {
if (this.active) return
this.active = true
this.offSettings = onAnySettingChanged(() => {
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
})
this.clockTimer = setInterval(() => this.requestShellRender(), 60_000)
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
console.log('[ios-preview] Main screen active')
}
pause(): void {
this.active = false
this.offSettings?.(); this.offSettings = null
if (this.clockTimer) clearInterval(this.clockTimer)
if (this.renderTimer) clearTimeout(this.renderTimer)
this.clockTimer = this.renderTimer = null
}
private configureWindow(window: ShellWindow): void {
this.compositor.configureSurface(window.surfaceId, {
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
})
this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
}
private relayout(): void {
const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}`
if (layout === this.lastLayout) return
this.lastLayout = layout
for (const window of shell.getWindows()) {
this.configureWindow(window)
window.relayout?.()
}
}
private async submit(id: string, planes: Plane[]): Promise<void> {
const image = flattenPlanes(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
this.scheduleFrame()
}
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
private scheduleFrame(): void {
if (!this.active || this.renderTimer !== null) return
// Coalesce window/chrome updates into one preview, capped at 30 fps.
this.renderTimer = setTimeout(() => {
this.renderTimer = null
try {
if (this.shellDirty) {
this.shellDirty = false
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
this.compositor.setUnderlayDim(1, shell.underlayDim())
}
const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
} catch (error) { this.fail(error) }
}, 33)
}
private fail(error: unknown): void {
const message = error instanceof Error ? error.stack ?? error.message : String(error)
console.error(`[ios-preview] ${message}`)
this.onError(error instanceof Error ? error.message : String(error))
}
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
this.inputQueue = this.inputQueue.then(async () => {
if (!this.active) return
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
await this.mirrorTap(nx, ny)
} else {
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
if (origin === 'ring' && type.startsWith('swipe-')) {
if (type === 'swipe-left' || type === 'swipe-right') return
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
}
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
await shell.receiveInput(event)
this.requestShellRender()
}
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
private async mirrorTap(nx: number, ny: number): Promise<void> {
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
const window = shell.foregroundWindow()
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) {
const target = shell.windowAtSidebarPoint(x, y)
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
return
}
if (window && !shell.hasOverlay()) {
const rect = appViewportRect(window.heightMode, window.appId)
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
shell.focusWindow(window.windowId)
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
}
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
this.requestShellRender()
}
launchApp(id: string): void {
const existing = shell.getWindows().find(window => window.appId === id)
if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return }
const surfaceId = `window:${id}`
let render = () => {}
const actions = { ...this.actions, requestRender: () => render() }
const layer = id === 'calculator'
? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp())
: new MenuLayer('Display', [
{ label: 'Change screen size', onSelect: () => this.cycleDisplayMode() },
{ label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } },
{ label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } },
], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true })
const app = createInProcessWindow({
appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display',
iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings',
closeable: true, actions, baseLayer: layer,
receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined,
submitFrame: planes => this.submit(surfaceId, planes),
setSurfaceVisible: visible => {
if (layer instanceof CalculatorLayer) layer.setForeground(visible)
this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame()
},
removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() },
onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() },
})
render = app.requestRender
if (layer instanceof CalculatorLayer) layer.requestRender = render
this.configureWindow(app.window)
shell.registerWindow(app.window)
shell.focusWindow(app.window.windowId)
app.requestRender(); this.requestShellRender()
}
cycleDisplayMode(): void {
const values = DISPLAY_MODE_VALUES
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
}
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
async typeIntoApp(): Promise<void> {
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
this.prompting = true
try {
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
if (result.result) shell.sendTextToForegroundWindow(result.text)
} catch (error) { this.fail(error) }
finally { this.prompting = false }
}
private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise<void> {
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' })
if (result.result) setting.set(result.text)
}
}
+7 -17
View File
@@ -1,12 +1,10 @@
import { GrayImage } from "./image";
declare const com: any;
declare const global: any;
import { rasterizeSvg } from "../native/svg-rasterizer";
/**
* Vector icons for window indicators (and anywhere else). SVGs are rendered
* once to a correctly sized grayscale bitmap by the Java IconRenderer (a
* small SVG subset: path/circle/rect/line/polyline) and cached here. To add
* once to a correctly sized grayscale bitmap by the platform rasterizer
* (Android IconRenderer or iOS SVGKit) and cached here. To add
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
* Noun Project glyphs also work.
@@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag
if (cached !== undefined) return cached;
let icon: GrayImage | null = null;
if (global.isAndroid) {
try {
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH);
if (bytes && bytes.length >= size * size) {
icon = new GrayImage(size, size, 0);
for (let i = 0; i < size * size; i++) {
icon.pixels[i] = bytes[i] & 0xff;
}
}
} catch (error) {
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
}
try {
icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH);
} catch (error) {
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
}
cache.set(key, icon);
return icon;
+75
View File
@@ -0,0 +1,75 @@
/** Platform-independent 8bpp surface composition for the local display. */
export type SurfaceRect = { x: number; y: number; width: number; height: number }
export type SurfaceConfiguration = SurfaceRect & {
zOrder: number
transparency: 'opaque' | 'color-key'
}
type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean }
export class SurfaceCompositor {
private readonly surfaces = new Map<string, Surface>()
private dimBelow = 0
private dimFactor = 1
private blanked = false
constructor(readonly width: number, readonly height: number) {
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
throw new Error('Invalid compositor size')
}
}
configureSurface(id: string, options: SurfaceConfiguration): void {
if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger)
|| options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry')
const previous = this.surfaces.get(id)
const pixels = previous?.width === options.width && previous.height === options.height
? previous.pixels : new Uint8Array(options.width * options.height)
this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true })
}
removeSurface(id: string): void { this.surfaces.delete(id) }
setSurfaceVisible(id: string, visible: boolean): void {
const surface = this.surfaces.get(id)
if (surface) surface.visible = visible
}
setUnderlayDim(belowZOrder: number, factor: number): void {
if (!Number.isFinite(factor)) throw new Error('Invalid dim factor')
this.dimBelow = belowZOrder
this.dimFactor = Math.max(0, Math.min(1, factor))
}
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
const surface = this.surfaces.get(id)
if (!surface) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) {
throw new Error('Invalid frame buffer or rectangle')
}
const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width)
const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height)
for (let y = top; y < bottom; y++) {
if (right <= left) break
const source = (y - rect.y) * rect.width + left - rect.x
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
}
}
composite(): Uint8Array {
const output = new Uint8Array(this.width * this.height)
if (this.blanked) return output
const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder)
for (const surface of surfaces) {
const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width)
const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height)
const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1
for (let y = top; y < bottom; y++) {
let source = (y - surface.y) * surface.width + left - surface.x
let target = y * this.width + left
for (let x = left; x < right; x++, source++, target++) {
const value = surface.pixels[source]
// Test the original value: opaque near-black must still cover the
// window beneath, even when dimming rounds it down to black.
if (surface.transparency === 'color-key' && value === 0) continue
output[target] = Math.round(value * dim)
}
}
}
return output
}
}
+21
View File
@@ -0,0 +1,21 @@
import { ImageSource } from '@nativescript/core'
import { GrayImage } from '../graphics/image'
declare const FaceclawGraphics: any
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
// NSData copies the buffer; the UIImage owns its pixels after this call.
const copy = new Uint8Array(pixels)
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
if (!native) throw new Error('Could not create preview image')
return new ImageSource(native)
}
export function rasterizeSvg(svg: string, size: number): GrayImage | null {
const data = FaceclawGraphics.renderSVGSize(svg, size)
if (!data) return null
const image = new GrayImage(size, size, 0)
image.pixels.set(new Uint8Array(interop.bufferFromData(data)))
return image
}
+28
View File
@@ -0,0 +1,28 @@
import { ApplicationSettings } from '@nativescript/core'
// Main-isolate settings for the initial iOS port. Worker synchronization will
// be added when worker-hosted apps are enabled on iOS.
const listeners = new Set<(key: string) => void>()
function changed(key: string): void {
setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0)
}
export function getStringSetting(key: string, fallback: string): string {
return ApplicationSettings.getString(key, fallback)
}
export function setStringSetting(key: string, value: string): void {
if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return
ApplicationSettings.setString(key, value)
changed(key)
}
export function getBooleanSetting(key: string, fallback: boolean): boolean {
return ApplicationSettings.getBoolean(key, fallback)
}
export function setBooleanSetting(key: string, value: boolean): void {
if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return
ApplicationSettings.setBoolean(key, value)
changed(key)
}
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
listeners.add(listener)
return () => { listeners.delete(listener) }
}
+14
View File
@@ -0,0 +1,14 @@
import { rasterizeSvg as render } from './ios-graphics'
export function rasterizeSvg(svg: string, size: number, strokeWidth: number) {
// Shared icon sources rely on the Android renderer's white, rounded
// stroke defaults. Supply those explicitly for a standards SVG renderer.
const source = svg.replace(/<svg\b([^>]*)>/, (_tag, attributes: string) => {
const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24',
stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth),
'stroke-linecap': 'round', 'stroke-linejoin': 'round' }
const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes))
.map(([key, value]) => ` ${key}="${value}"`).join('')
return `<svg${attributes}${missing}>`
})
return render(source, size)
}
+9
View File
@@ -0,0 +1,9 @@
import { GrayImage } from '../graphics/image'
declare const com: any
export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null {
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth)
if (!bytes || bytes.length < size * size) return null
const image = new GrayImage(size, size, 0)
for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff
return image
}
+141
View File
@@ -0,0 +1,141 @@
import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core'
import { IosPreviewController } from '../g2/ios-preview-controller'
import { PhoneGestureRecognizer } from './phone-gestures'
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
export function createMainPage(): Page {
const page = new Page()
page.actionBarHidden = true
const root = new GridLayout()
root.rows = 'auto,*'
page.content = root
const header = new GridLayout()
header.columns = '*,auto'; header.padding = '12 20'
const title = new Label()
title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold'
header.addChild(title)
const state = new Label()
state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle'
GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header)
const body = new GridLayout()
GridLayout.setRow(body, 1); root.addChild(body)
const mirror = new Image()
mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black')
mirror.accessibilityIdentifier = 'glasses-preview'
mirror.accessibilityLabel = 'Interactive glasses screen preview'
mirror.verticalAlignment = 'middle'; body.addChild(mirror)
const controls = new GridLayout()
controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls)
const tabs = new GridLayout()
tabs.columns = '*,*,*'; controls.addChild(tabs)
const pad = new GridLayout()
pad.borderRadius = 18; pad.backgroundColor = new Color('#202522')
pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center'
pad.accessibilityIdentifier = 'gesture-pad'
GridLayout.setRow(pad, 1); controls.addChild(pad)
const focus = new Label()
focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true
focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle'
focus.isUserInteractionEnabled = false; pad.addChild(focus)
const legend = new Label()
legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center'
legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom'
legend.isUserInteractionEnabled = false; pad.addChild(legend)
const settings = new StackLayout()
settings.verticalAlignment = 'middle'
GridLayout.setRow(settings, 1); controls.addChild(settings)
const errorLabel = new Label()
errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse'
settings.addChild(errorLabel)
let selectedTab: 'watch' | 'ring' | 'settings' = 'watch'
const controller = new IosPreviewController((image, label) => {
mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label
}, message => {
errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings')
})
const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch'))
const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => {
const size = mirror.getActualSize()
if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height)
})
const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => {
if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return
recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() })
}
pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput))
function button(text: string, action: () => void): Button {
const view = new Button()
view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action)
return view
}
const tabButtons: Button[] = []
function selectTab(tab: typeof selectedTab): void {
padInput.cancel(); selectedTab = tab
pad.visibility = tab === 'settings' ? 'collapse' : 'visible'
settings.visibility = tab === 'settings' ? 'visible' : 'collapse'
for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5
focus.text = tab === 'ring' ? 'Ring' : 'Watch controls'
legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu'
layout()
}
for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) {
const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab))
GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view)
}
const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode())
settings.addChild(sizeButton)
settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green')))
const help = new Label()
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
const footer = new GridLayout()
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
const menu = button('Menu', () => {
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
})
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
for (const [index, view] of [back, menu, keyboard].entries()) {
view.padding = '16 8'; view.fontSize = 14
GridLayout.setColumn(view, index); footer.addChild(view)
}
let lastLayout = ''
function layout(): void {
const size = root.getActualSize()
if (size.width <= 0 || size.height <= 0) return
const key = `${size.width}:${size.height}:${selectedTab}`
if (key === lastLayout) return
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
const landscape = size.width > size.height
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
mirror.width = width; mirror.height = width * 3 / 4
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
}
// iOS emits layoutChanged inside its layout pass. Defer mutations so
// NativeScript does not clear our new layout request at the end of that pass.
let layoutTimer: ReturnType<typeof setTimeout> | null = null
root.on('layoutChanged', () => {
if (layoutTimer !== null) return
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
})
const pause = () => {
if (layoutTimer !== null) clearTimeout(layoutTimer)
layoutTimer = null
padInput.cancel(); mirrorInput.cancel(); controller.pause()
}
const resume = () => controller.resume()
let offSettings: (() => void) | null = null
page.on('loaded', () => {
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
selectTab(selectedTab); controller.resume()
})
page.on('unloaded', () => {
pause(); offSettings?.(); offSettings = null
Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume)
})
return page
}
+70
View File
@@ -0,0 +1,70 @@
/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */
declare function setTimeout(callback: () => void, ms: number): number
declare function clearTimeout(id: number): void
export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release'
| 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right'
export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number }
export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void }
const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType<typeof setTimeout>) }
export class PhoneGestureRecognizer {
private start: PhoneTouch | null = null
private hold: unknown = null
private tap: unknown = null
private lastTap: { x: number; y: number } | null = null
private holding = false
private moved = false
private multi = false
private secondTap = false
constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void,
private readonly timer: GestureClock = clock) {}
touch(event: PhoneTouch): void {
if (event.action === 'cancel') { this.cancel(); return }
if (event.action === 'down' && !this.start) {
this.start = event
this.holding = this.moved = this.multi = false
this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36
if (this.secondTap) { this.clearTap() }
else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) }
this.hold = this.timer.set(() => {
this.hold = null
if (!this.start || this.moved || this.multi) return
this.holding = true
this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y)
}, 500)
}
if (!this.start) return
if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() }
const dx = event.x - this.start.x, dy = event.y - this.start.y
if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() }
if (event.action !== 'up') return
// Some platforms emit an up for each finger. Complete the multi-touch
// gesture only once, and suppress the trailing one-finger up events.
this.clearHold()
const start = this.start
this.start = null
if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return }
if (this.multi) { this.emit('double-tap', start.x, start.y); return }
if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) {
this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y)
} else if (!this.moved) {
if (this.secondTap) this.emit('double-tap', start.x, start.y)
else {
this.lastTap = { x: start.x, y: start.y }
this.tap = this.timer.set(() => {
const tap = this.lastTap
this.tap = null; this.lastTap = null
if (tap) this.emit('tap', tap.x, tap.y)
}, 280)
}
}
}
cancel(): void {
this.clearHold(); this.clearTap()
if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y)
this.start = null; this.holding = false
}
private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null }
private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null }
}
+6 -3
View File
@@ -144,6 +144,8 @@ export type ShellWindow = {
};
export type ShellConfig = {
/** Hosts without microphone support hide and reject voice entry points. */
voiceInputEnabled?: boolean;
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
actions: LayerActions;
getScreenTimeoutMs: () => number | null;
@@ -901,7 +903,7 @@ class Shell {
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
if (this.voiceDialogPending) return;
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
this.voiceDialogPending = true;
void (async () => {
let ready = true;
@@ -1005,6 +1007,7 @@ class Shell {
* the dialog finishes on click instead of long-press-release.
*/
startVoiceInput(): void {
if (this.config.voiceInputEnabled === false) return;
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
// The transcript is aimed at the window whose menu requested it; the menu
// entry point defaults the highlight to Type Into App.
@@ -1334,7 +1337,7 @@ class Shell {
ctx.stack.pop();
},
},
{
...(this.config.voiceInputEnabled === false ? [] : [{
label: "Voice input",
onSelect: (ctx) => {
// The transcript is aimed at the foreground window, so keep focus
@@ -1343,7 +1346,7 @@ class Shell {
ctx.stack.pop();
this.startVoiceInput();
},
},
}]),
);
items.push({
label: "Debug",