forked from Blendman974/faceclaw
iOS porting: Main screen and compositor
This commit is contained in:
1 parent
61b3bf8b7c
commit
99793a37a7
24 files changed
+1124
-70
No files matched your search
@@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer {
|
||||
constructor(
|
||||
private readonly coordinator: MathCoordinator,
|
||||
private readonly actions: LayerActions,
|
||||
private readonly textInput?: () => Promise<void>,
|
||||
) {
|
||||
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
||||
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
}
|
||||
|
||||
private get mode(): CalculatorMode {
|
||||
@@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer {
|
||||
}
|
||||
|
||||
private refreshListening(): void {
|
||||
if (this.textInput) return;
|
||||
if (!this.foreground) return;
|
||||
if (this.listening === "continuous") {
|
||||
if (!this.continuousActive) {
|
||||
@@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer {
|
||||
* that did not.
|
||||
*/
|
||||
beginDictation(): void {
|
||||
if (this.textInput) { void this.textInput(); return; }
|
||||
if (this.dictating) return;
|
||||
this.dictating = true;
|
||||
void this.actions.startVoiceCapture(true);
|
||||
@@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer {
|
||||
bodyBottom: number,
|
||||
): void {
|
||||
const prompt =
|
||||
this.listening === "continuous"
|
||||
this.textInput ? "Tap to type a problem on your phone"
|
||||
: this.listening === "continuous"
|
||||
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
|
||||
: "Tap to say a problem — or use Voice input to type one";
|
||||
const lines = wrapText(font, prompt, width - 96);
|
||||
@@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer {
|
||||
|
||||
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||
if (!this.lastInput) return;
|
||||
const label = "Heard";
|
||||
const label = this.textInput ? "Entered" : "Heard";
|
||||
image.drawText(font, 24, y, label, 110);
|
||||
const x = 24 + font.measureText(label) + 10;
|
||||
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
|
||||
@@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer {
|
||||
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, scrollTarget],
|
||||
[GESTURE_CLICK, "mic"],
|
||||
[GESTURE_CLICK, this.textInput ? "type" : "mic"],
|
||||
[GESTURE_DOUBLE_CLICK, "back"],
|
||||
]);
|
||||
}
|
||||
|
||||
+3
-46
@@ -1,47 +1,4 @@
|
||||
import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core'
|
||||
import { bytesToHex, hexToBytes } from './util/hex-util'
|
||||
import { Application } from '@nativescript/core'
|
||||
import { createMainPage } from './phone-ui/main-page.ios'
|
||||
|
||||
// Milestone 1: exercise native UI, TypeScript, and an existing shared utility
|
||||
// without loading the Android services used by the full glasses application.
|
||||
Application.run({
|
||||
create: () => {
|
||||
const page = new Page()
|
||||
page.actionBarHidden = true
|
||||
const scroll = new ScrollView()
|
||||
const content = new StackLayout()
|
||||
content.padding = 28
|
||||
content.verticalAlignment = 'middle'
|
||||
|
||||
const label = (text: string, fontSize = 17) => {
|
||||
const view = new Label()
|
||||
view.text = text
|
||||
view.fontSize = fontSize
|
||||
view.textWrap = true
|
||||
view.marginBottom = 20
|
||||
content.addChild(view)
|
||||
return view
|
||||
}
|
||||
|
||||
label('Faceclaw', 36)
|
||||
label('iOS port · First launch', 24)
|
||||
label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`)
|
||||
label('Glasses pairing and the Android features are still to be ported.')
|
||||
const status = label('Tap below to test native input and shared TypeScript code.')
|
||||
const button = new Button()
|
||||
button.text = 'Run shared-code check'
|
||||
button.accessibilityIdentifier = 'shared-code-check'
|
||||
let checks = 0
|
||||
button.on('tap', () => {
|
||||
const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77'))
|
||||
const passed = result === '46616365636C6177'
|
||||
checks += 1
|
||||
status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.`
|
||||
console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`)
|
||||
})
|
||||
content.addChild(button)
|
||||
scroll.content = content
|
||||
page.content = scroll
|
||||
page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded'))
|
||||
return page
|
||||
},
|
||||
})
|
||||
Application.run({ create: createMainPage })
|
||||
@@ -0,0 +1,220 @@
|
||||
import { Dialogs, type ImageSource } from '@nativescript/core'
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||
import { CalculatorLayer } from '../apps/calculator/calculator'
|
||||
import { MathCoordinator } from '../apps/calculator/math/coordinator'
|
||||
import { SurfaceCompositor } from '../graphics/surface-compositor'
|
||||
import { flattenPlanes, type Plane } from '../graphics/plane'
|
||||
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||
import { previewPixels } from '../native/ios-graphics'
|
||||
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
|
||||
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||
import { MenuLayer } from '../ui/menu'
|
||||
import { createInProcessWindow } from '../ui/shell/in-process-window'
|
||||
import { shell, type ShellWindow } from '../ui/shell/shell'
|
||||
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||
previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
|
||||
/** Local display host: shared shell/windows, no device-service initialization. */
|
||||
export class IosPreviewController {
|
||||
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
|
||||
private renderTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private clockTimer: ReturnType<typeof setInterval> | null = null
|
||||
private offSettings: (() => void) | null = null
|
||||
private active = false
|
||||
private shellDirty = true
|
||||
private inputQueue: Promise<void> = Promise.resolve()
|
||||
private lastLayout = ''
|
||||
private prompting = false
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
startTextSettingEdit: setting => this.editSetting(setting),
|
||||
startTextSettingsEdit: async (settings, _title, finished) => {
|
||||
for (const setting of settings) await this.editSetting(setting)
|
||||
finished?.()
|
||||
},
|
||||
}
|
||||
|
||||
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
|
||||
private readonly onError: (message: string) => void) {
|
||||
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
voiceInputEnabled: false,
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => this.requestShellRender(),
|
||||
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
|
||||
})
|
||||
const launcher = createLauncherWindow({
|
||||
actions: this.actions,
|
||||
apps: () => [
|
||||
{ appId: 'calculator', label: 'Calculator', icon: 'calculator' },
|
||||
{ appId: 'display', label: 'Display', icon: 'settings' },
|
||||
],
|
||||
launchApp: id => this.launchApp(id),
|
||||
uninstallApp: () => {},
|
||||
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
|
||||
setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() },
|
||||
})
|
||||
this.configureWindow(launcher)
|
||||
shell.registerWindow(launcher)
|
||||
shell.wake('window')
|
||||
shell.focusWindow(launcher.windowId)
|
||||
}
|
||||
|
||||
resume(): void {
|
||||
if (this.active) return
|
||||
this.active = true
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
})
|
||||
this.clockTimer = setInterval(() => this.requestShellRender(), 60_000)
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
console.log('[ios-preview] Main screen active')
|
||||
}
|
||||
pause(): void {
|
||||
this.active = false
|
||||
this.offSettings?.(); this.offSettings = null
|
||||
if (this.clockTimer) clearInterval(this.clockTimer)
|
||||
if (this.renderTimer) clearTimeout(this.renderTimer)
|
||||
this.clockTimer = this.renderTimer = null
|
||||
}
|
||||
private configureWindow(window: ShellWindow): void {
|
||||
this.compositor.configureSurface(window.surfaceId, {
|
||||
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
|
||||
})
|
||||
this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
|
||||
}
|
||||
private relayout(): void {
|
||||
const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}`
|
||||
if (layout === this.lastLayout) return
|
||||
this.lastLayout = layout
|
||||
for (const window of shell.getWindows()) {
|
||||
this.configureWindow(window)
|
||||
window.relayout?.()
|
||||
}
|
||||
}
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
const image = flattenPlanes(planes)
|
||||
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
|
||||
this.scheduleFrame()
|
||||
}
|
||||
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
|
||||
private scheduleFrame(): void {
|
||||
if (!this.active || this.renderTimer !== null) return
|
||||
// Coalesce window/chrome updates into one preview, capped at 30 fps.
|
||||
this.renderTimer = setTimeout(() => {
|
||||
this.renderTimer = null
|
||||
try {
|
||||
if (this.shellDirty) {
|
||||
this.shellDirty = false
|
||||
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
|
||||
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
|
||||
this.compositor.setUnderlayDim(1, shell.underlayDim())
|
||||
}
|
||||
const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green')
|
||||
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||
} catch (error) { this.fail(error) }
|
||||
}, 33)
|
||||
}
|
||||
private fail(error: unknown): void {
|
||||
const message = error instanceof Error ? error.stack ?? error.message : String(error)
|
||||
console.error(`[ios-preview] ${message}`)
|
||||
this.onError(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
|
||||
this.inputQueue = this.inputQueue.then(async () => {
|
||||
if (!this.active) return
|
||||
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
|
||||
await this.mirrorTap(nx, ny)
|
||||
} else {
|
||||
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
|
||||
if (origin === 'ring' && type.startsWith('swipe-')) {
|
||||
if (type === 'swipe-left' || type === 'swipe-right') return
|
||||
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
|
||||
}
|
||||
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
|
||||
await shell.receiveInput(event)
|
||||
this.requestShellRender()
|
||||
}
|
||||
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
|
||||
}).catch(error => this.fail(error))
|
||||
}
|
||||
private async mirrorTap(nx: number, ny: number): Promise<void> {
|
||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||
const window = shell.foregroundWindow()
|
||||
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) {
|
||||
const target = shell.windowAtSidebarPoint(x, y)
|
||||
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
|
||||
return
|
||||
}
|
||||
if (window && !shell.hasOverlay()) {
|
||||
const rect = appViewportRect(window.heightMode, window.appId)
|
||||
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
|
||||
shell.focusWindow(window.windowId)
|
||||
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
|
||||
}
|
||||
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
|
||||
this.requestShellRender()
|
||||
}
|
||||
launchApp(id: string): void {
|
||||
const existing = shell.getWindows().find(window => window.appId === id)
|
||||
if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return }
|
||||
const surfaceId = `window:${id}`
|
||||
let render = () => {}
|
||||
const actions = { ...this.actions, requestRender: () => render() }
|
||||
const layer = id === 'calculator'
|
||||
? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp())
|
||||
: new MenuLayer('Display', [
|
||||
{ label: 'Change screen size', onSelect: () => this.cycleDisplayMode() },
|
||||
{ label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } },
|
||||
{ label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } },
|
||||
], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true })
|
||||
const app = createInProcessWindow({
|
||||
appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display',
|
||||
iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings',
|
||||
closeable: true, actions, baseLayer: layer,
|
||||
receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined,
|
||||
submitFrame: planes => this.submit(surfaceId, planes),
|
||||
setSurfaceVisible: visible => {
|
||||
if (layer instanceof CalculatorLayer) layer.setForeground(visible)
|
||||
this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame()
|
||||
},
|
||||
removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() },
|
||||
onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() },
|
||||
})
|
||||
render = app.requestRender
|
||||
if (layer instanceof CalculatorLayer) layer.requestRender = render
|
||||
this.configureWindow(app.window)
|
||||
shell.registerWindow(app.window)
|
||||
shell.focusWindow(app.window.windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
}
|
||||
cycleDisplayMode(): void {
|
||||
const values = DISPLAY_MODE_VALUES
|
||||
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
|
||||
}
|
||||
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
|
||||
async typeIntoApp(): Promise<void> {
|
||||
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
|
||||
this.prompting = true
|
||||
try {
|
||||
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
|
||||
message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
|
||||
if (result.result) shell.sendTextToForegroundWindow(result.text)
|
||||
} catch (error) { this.fail(error) }
|
||||
finally { this.prompting = false }
|
||||
}
|
||||
private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise<void> {
|
||||
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' })
|
||||
if (result.result) setting.set(result.text)
|
||||
}
|
||||
}
|
||||
+7
-17
@@ -1,12 +1,10 @@
|
||||
import { GrayImage } from "./image";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
import { rasterizeSvg } from "../native/svg-rasterizer";
|
||||
|
||||
/**
|
||||
* Vector icons for window indicators (and anywhere else). SVGs are rendered
|
||||
* once to a correctly sized grayscale bitmap by the Java IconRenderer (a
|
||||
* small SVG subset: path/circle/rect/line/polyline) and cached here. To add
|
||||
* once to a correctly sized grayscale bitmap by the platform rasterizer
|
||||
* (Android IconRenderer or iOS SVGKit) and cached here. To add
|
||||
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
|
||||
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
|
||||
* Noun Project glyphs also work.
|
||||
@@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
let icon: GrayImage | null = null;
|
||||
if (global.isAndroid) {
|
||||
try {
|
||||
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH);
|
||||
if (bytes && bytes.length >= size * size) {
|
||||
icon = new GrayImage(size, size, 0);
|
||||
for (let i = 0; i < size * size; i++) {
|
||||
icon.pixels[i] = bytes[i] & 0xff;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
||||
}
|
||||
try {
|
||||
icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH);
|
||||
} catch (error) {
|
||||
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
||||
}
|
||||
cache.set(key, icon);
|
||||
return icon;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/** Platform-independent 8bpp surface composition for the local display. */
|
||||
export type SurfaceRect = { x: number; y: number; width: number; height: number }
|
||||
export type SurfaceConfiguration = SurfaceRect & {
|
||||
zOrder: number
|
||||
transparency: 'opaque' | 'color-key'
|
||||
}
|
||||
type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean }
|
||||
|
||||
export class SurfaceCompositor {
|
||||
private readonly surfaces = new Map<string, Surface>()
|
||||
private dimBelow = 0
|
||||
private dimFactor = 1
|
||||
private blanked = false
|
||||
constructor(readonly width: number, readonly height: number) {
|
||||
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
|
||||
throw new Error('Invalid compositor size')
|
||||
}
|
||||
}
|
||||
configureSurface(id: string, options: SurfaceConfiguration): void {
|
||||
if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger)
|
||||
|| options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry')
|
||||
const previous = this.surfaces.get(id)
|
||||
const pixels = previous?.width === options.width && previous.height === options.height
|
||||
? previous.pixels : new Uint8Array(options.width * options.height)
|
||||
this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true })
|
||||
}
|
||||
removeSurface(id: string): void { this.surfaces.delete(id) }
|
||||
setSurfaceVisible(id: string, visible: boolean): void {
|
||||
const surface = this.surfaces.get(id)
|
||||
if (surface) surface.visible = visible
|
||||
}
|
||||
setUnderlayDim(belowZOrder: number, factor: number): void {
|
||||
if (!Number.isFinite(factor)) throw new Error('Invalid dim factor')
|
||||
this.dimBelow = belowZOrder
|
||||
this.dimFactor = Math.max(0, Math.min(1, factor))
|
||||
}
|
||||
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
|
||||
const surface = this.surfaces.get(id)
|
||||
if (!surface) throw new Error(`Unknown surface: ${id}`)
|
||||
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|
||||
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) {
|
||||
throw new Error('Invalid frame buffer or rectangle')
|
||||
}
|
||||
const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width)
|
||||
const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height)
|
||||
for (let y = top; y < bottom; y++) {
|
||||
if (right <= left) break
|
||||
const source = (y - rect.y) * rect.width + left - rect.x
|
||||
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
|
||||
}
|
||||
}
|
||||
composite(): Uint8Array {
|
||||
const output = new Uint8Array(this.width * this.height)
|
||||
if (this.blanked) return output
|
||||
const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder)
|
||||
for (const surface of surfaces) {
|
||||
const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width)
|
||||
const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height)
|
||||
const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1
|
||||
for (let y = top; y < bottom; y++) {
|
||||
let source = (y - surface.y) * surface.width + left - surface.x
|
||||
let target = y * this.width + left
|
||||
for (let x = left; x < right; x++, source++, target++) {
|
||||
const value = surface.pixels[source]
|
||||
// Test the original value: opaque near-black must still cover the
|
||||
// window beneath, even when dimming rounds it down to black.
|
||||
if (surface.transparency === 'color-key' && value === 0) continue
|
||||
output[target] = Math.round(value * dim)
|
||||
}
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { ImageSource } from '@nativescript/core'
|
||||
import { GrayImage } from '../graphics/image'
|
||||
|
||||
declare const FaceclawGraphics: any
|
||||
|
||||
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
|
||||
// NSData copies the buffer; the UIImage owns its pixels after this call.
|
||||
const copy = new Uint8Array(pixels)
|
||||
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
|
||||
if (!native) throw new Error('Could not create preview image')
|
||||
return new ImageSource(native)
|
||||
}
|
||||
|
||||
export function rasterizeSvg(svg: string, size: number): GrayImage | null {
|
||||
const data = FaceclawGraphics.renderSVGSize(svg, size)
|
||||
if (!data) return null
|
||||
const image = new GrayImage(size, size, 0)
|
||||
image.pixels.set(new Uint8Array(interop.bufferFromData(data)))
|
||||
return image
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { ApplicationSettings } from '@nativescript/core'
|
||||
|
||||
// Main-isolate settings for the initial iOS port. Worker synchronization will
|
||||
// be added when worker-hosted apps are enabled on iOS.
|
||||
const listeners = new Set<(key: string) => void>()
|
||||
function changed(key: string): void {
|
||||
setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0)
|
||||
}
|
||||
export function getStringSetting(key: string, fallback: string): string {
|
||||
return ApplicationSettings.getString(key, fallback)
|
||||
}
|
||||
export function setStringSetting(key: string, value: string): void {
|
||||
if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return
|
||||
ApplicationSettings.setString(key, value)
|
||||
changed(key)
|
||||
}
|
||||
export function getBooleanSetting(key: string, fallback: boolean): boolean {
|
||||
return ApplicationSettings.getBoolean(key, fallback)
|
||||
}
|
||||
export function setBooleanSetting(key: string, value: boolean): void {
|
||||
if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return
|
||||
ApplicationSettings.setBoolean(key, value)
|
||||
changed(key)
|
||||
}
|
||||
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
|
||||
listeners.add(listener)
|
||||
return () => { listeners.delete(listener) }
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { rasterizeSvg as render } from './ios-graphics'
|
||||
export function rasterizeSvg(svg: string, size: number, strokeWidth: number) {
|
||||
// Shared icon sources rely on the Android renderer's white, rounded
|
||||
// stroke defaults. Supply those explicitly for a standards SVG renderer.
|
||||
const source = svg.replace(/<svg\b([^>]*)>/, (_tag, attributes: string) => {
|
||||
const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24',
|
||||
stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth),
|
||||
'stroke-linecap': 'round', 'stroke-linejoin': 'round' }
|
||||
const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes))
|
||||
.map(([key, value]) => ` ${key}="${value}"`).join('')
|
||||
return `<svg${attributes}${missing}>`
|
||||
})
|
||||
return render(source, size)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { GrayImage } from '../graphics/image'
|
||||
declare const com: any
|
||||
export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null {
|
||||
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth)
|
||||
if (!bytes || bytes.length < size * size) return null
|
||||
const image = new GrayImage(size, size, 0)
|
||||
for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff
|
||||
return image
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core'
|
||||
import { IosPreviewController } from '../g2/ios-preview-controller'
|
||||
import { PhoneGestureRecognizer } from './phone-gestures'
|
||||
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
|
||||
|
||||
export function createMainPage(): Page {
|
||||
const page = new Page()
|
||||
page.actionBarHidden = true
|
||||
const root = new GridLayout()
|
||||
root.rows = 'auto,*'
|
||||
page.content = root
|
||||
const header = new GridLayout()
|
||||
header.columns = '*,auto'; header.padding = '12 20'
|
||||
const title = new Label()
|
||||
title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold'
|
||||
header.addChild(title)
|
||||
const state = new Label()
|
||||
state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle'
|
||||
GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header)
|
||||
const body = new GridLayout()
|
||||
GridLayout.setRow(body, 1); root.addChild(body)
|
||||
const mirror = new Image()
|
||||
mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black')
|
||||
mirror.accessibilityIdentifier = 'glasses-preview'
|
||||
mirror.accessibilityLabel = 'Interactive glasses screen preview'
|
||||
mirror.verticalAlignment = 'middle'; body.addChild(mirror)
|
||||
const controls = new GridLayout()
|
||||
controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls)
|
||||
const tabs = new GridLayout()
|
||||
tabs.columns = '*,*,*'; controls.addChild(tabs)
|
||||
const pad = new GridLayout()
|
||||
pad.borderRadius = 18; pad.backgroundColor = new Color('#202522')
|
||||
pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center'
|
||||
pad.accessibilityIdentifier = 'gesture-pad'
|
||||
GridLayout.setRow(pad, 1); controls.addChild(pad)
|
||||
const focus = new Label()
|
||||
focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true
|
||||
focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle'
|
||||
focus.isUserInteractionEnabled = false; pad.addChild(focus)
|
||||
const legend = new Label()
|
||||
legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center'
|
||||
legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom'
|
||||
legend.isUserInteractionEnabled = false; pad.addChild(legend)
|
||||
const settings = new StackLayout()
|
||||
settings.verticalAlignment = 'middle'
|
||||
GridLayout.setRow(settings, 1); controls.addChild(settings)
|
||||
const errorLabel = new Label()
|
||||
errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse'
|
||||
settings.addChild(errorLabel)
|
||||
let selectedTab: 'watch' | 'ring' | 'settings' = 'watch'
|
||||
const controller = new IosPreviewController((image, label) => {
|
||||
mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label
|
||||
}, message => {
|
||||
errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings')
|
||||
})
|
||||
const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch'))
|
||||
const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => {
|
||||
const size = mirror.getActualSize()
|
||||
if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height)
|
||||
})
|
||||
const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => {
|
||||
if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return
|
||||
recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() })
|
||||
}
|
||||
pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput))
|
||||
function button(text: string, action: () => void): Button {
|
||||
const view = new Button()
|
||||
view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action)
|
||||
return view
|
||||
}
|
||||
const tabButtons: Button[] = []
|
||||
function selectTab(tab: typeof selectedTab): void {
|
||||
padInput.cancel(); selectedTab = tab
|
||||
pad.visibility = tab === 'settings' ? 'collapse' : 'visible'
|
||||
settings.visibility = tab === 'settings' ? 'visible' : 'collapse'
|
||||
for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5
|
||||
focus.text = tab === 'ring' ? 'Ring' : 'Watch controls'
|
||||
legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu'
|
||||
layout()
|
||||
}
|
||||
for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) {
|
||||
const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab))
|
||||
GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view)
|
||||
}
|
||||
const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode())
|
||||
settings.addChild(sizeButton)
|
||||
settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green')))
|
||||
const help = new Label()
|
||||
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
|
||||
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
|
||||
const footer = new GridLayout()
|
||||
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
|
||||
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
|
||||
const menu = button('Menu', () => {
|
||||
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
|
||||
})
|
||||
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
|
||||
for (const [index, view] of [back, menu, keyboard].entries()) {
|
||||
view.padding = '16 8'; view.fontSize = 14
|
||||
GridLayout.setColumn(view, index); footer.addChild(view)
|
||||
}
|
||||
let lastLayout = ''
|
||||
function layout(): void {
|
||||
const size = root.getActualSize()
|
||||
if (size.width <= 0 || size.height <= 0) return
|
||||
const key = `${size.width}:${size.height}:${selectedTab}`
|
||||
if (key === lastLayout) return
|
||||
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
|
||||
const landscape = size.width > size.height
|
||||
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
|
||||
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
|
||||
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
|
||||
mirror.width = width; mirror.height = width * 3 / 4
|
||||
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
|
||||
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
|
||||
}
|
||||
// iOS emits layoutChanged inside its layout pass. Defer mutations so
|
||||
// NativeScript does not clear our new layout request at the end of that pass.
|
||||
let layoutTimer: ReturnType<typeof setTimeout> | null = null
|
||||
root.on('layoutChanged', () => {
|
||||
if (layoutTimer !== null) return
|
||||
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
|
||||
})
|
||||
const pause = () => {
|
||||
if (layoutTimer !== null) clearTimeout(layoutTimer)
|
||||
layoutTimer = null
|
||||
padInput.cancel(); mirrorInput.cancel(); controller.pause()
|
||||
}
|
||||
const resume = () => controller.resume()
|
||||
let offSettings: (() => void) | null = null
|
||||
page.on('loaded', () => {
|
||||
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
|
||||
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
|
||||
selectTab(selectedTab); controller.resume()
|
||||
})
|
||||
page.on('unloaded', () => {
|
||||
pause(); offSettings?.(); offSettings = null
|
||||
Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume)
|
||||
})
|
||||
return page
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */
|
||||
declare function setTimeout(callback: () => void, ms: number): number
|
||||
declare function clearTimeout(id: number): void
|
||||
export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release'
|
||||
| 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right'
|
||||
export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number }
|
||||
export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void }
|
||||
const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType<typeof setTimeout>) }
|
||||
|
||||
export class PhoneGestureRecognizer {
|
||||
private start: PhoneTouch | null = null
|
||||
private hold: unknown = null
|
||||
private tap: unknown = null
|
||||
private lastTap: { x: number; y: number } | null = null
|
||||
private holding = false
|
||||
private moved = false
|
||||
private multi = false
|
||||
private secondTap = false
|
||||
constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void,
|
||||
private readonly timer: GestureClock = clock) {}
|
||||
|
||||
touch(event: PhoneTouch): void {
|
||||
if (event.action === 'cancel') { this.cancel(); return }
|
||||
if (event.action === 'down' && !this.start) {
|
||||
this.start = event
|
||||
this.holding = this.moved = this.multi = false
|
||||
this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36
|
||||
if (this.secondTap) { this.clearTap() }
|
||||
else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) }
|
||||
this.hold = this.timer.set(() => {
|
||||
this.hold = null
|
||||
if (!this.start || this.moved || this.multi) return
|
||||
this.holding = true
|
||||
this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y)
|
||||
}, 500)
|
||||
}
|
||||
if (!this.start) return
|
||||
if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() }
|
||||
const dx = event.x - this.start.x, dy = event.y - this.start.y
|
||||
if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() }
|
||||
if (event.action !== 'up') return
|
||||
// Some platforms emit an up for each finger. Complete the multi-touch
|
||||
// gesture only once, and suppress the trailing one-finger up events.
|
||||
this.clearHold()
|
||||
const start = this.start
|
||||
this.start = null
|
||||
if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return }
|
||||
if (this.multi) { this.emit('double-tap', start.x, start.y); return }
|
||||
if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) {
|
||||
this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y)
|
||||
} else if (!this.moved) {
|
||||
if (this.secondTap) this.emit('double-tap', start.x, start.y)
|
||||
else {
|
||||
this.lastTap = { x: start.x, y: start.y }
|
||||
this.tap = this.timer.set(() => {
|
||||
const tap = this.lastTap
|
||||
this.tap = null; this.lastTap = null
|
||||
if (tap) this.emit('tap', tap.x, tap.y)
|
||||
}, 280)
|
||||
}
|
||||
}
|
||||
}
|
||||
cancel(): void {
|
||||
this.clearHold(); this.clearTap()
|
||||
if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y)
|
||||
this.start = null; this.holding = false
|
||||
}
|
||||
private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null }
|
||||
private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null }
|
||||
}
|
||||
@@ -144,6 +144,8 @@ export type ShellWindow = {
|
||||
};
|
||||
|
||||
export type ShellConfig = {
|
||||
/** Hosts without microphone support hide and reject voice entry points. */
|
||||
voiceInputEnabled?: boolean;
|
||||
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
|
||||
actions: LayerActions;
|
||||
getScreenTimeoutMs: () => number | null;
|
||||
@@ -901,7 +903,7 @@ class Shell {
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
if (this.voiceDialogPending) return;
|
||||
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
|
||||
this.voiceDialogPending = true;
|
||||
void (async () => {
|
||||
let ready = true;
|
||||
@@ -1005,6 +1007,7 @@ class Shell {
|
||||
* the dialog finishes on click instead of long-press-release.
|
||||
*/
|
||||
startVoiceInput(): void {
|
||||
if (this.config.voiceInputEnabled === false) return;
|
||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
||||
// The transcript is aimed at the window whose menu requested it; the menu
|
||||
// entry point defaults the highlight to Type Into App.
|
||||
@@ -1334,7 +1337,7 @@ class Shell {
|
||||
ctx.stack.pop();
|
||||
},
|
||||
},
|
||||
{
|
||||
...(this.config.voiceInputEnabled === false ? [] : [{
|
||||
label: "Voice input",
|
||||
onSelect: (ctx) => {
|
||||
// The transcript is aimed at the foreground window, so keep focus
|
||||
@@ -1343,7 +1346,7 @@ class Shell {
|
||||
ctx.stack.pop();
|
||||
this.startVoiceInput();
|
||||
},
|
||||
},
|
||||
}]),
|
||||
);
|
||||
items.push({
|
||||
label: "Debug",
|
||||
|
||||
Reference in new issue
Block a user