forked from Blendman974/faceclaw
iOS porting: Main screen and compositor
This commit is contained in:
1 parent
61b3bf8b7c
commit
99793a37a7
24 files changed
+1124
-70
No files matched your search
@@ -23,6 +23,8 @@
|
|||||||
<key>UISupportedInterfaceOrientations</key>
|
<key>UISupportedInterfaceOrientations</key>
|
||||||
<array>
|
<array>
|
||||||
<string>UIInterfaceOrientationPortrait</string>
|
<string>UIInterfaceOrientationPortrait</string>
|
||||||
|
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||||
|
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||||
</array>
|
</array>
|
||||||
<key>UIRequiresFullScreen</key>
|
<key>UIRequiresFullScreen</key>
|
||||||
<true/>
|
<true/>
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
platform :ios, '15.1'
|
||||||
|
pod 'SVGKit', '3.0.0'
|
||||||
|
|
||||||
|
# SVGKit 3's physical-DPI lookup asserts on arm64 simulators. Our icons use
|
||||||
|
# only viewBox/pixel units, so that lookup is irrelevant to their rendering.
|
||||||
|
post_install do |installer|
|
||||||
|
installer.pods_project.targets.each do |target|
|
||||||
|
next unless target.name == 'SVGKit'
|
||||||
|
target.build_configurations.each do |config|
|
||||||
|
config.build_settings['ENABLE_NS_ASSERTIONS'] = 'NO'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#import <UIKit/UIKit.h>
|
||||||
|
|
||||||
|
@interface FaceclawGraphics : NSObject
|
||||||
|
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
|
||||||
|
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
|
||||||
|
@end
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#import "FaceclawGraphics.h"
|
||||||
|
#import <SVGKit/SVGKit.h>
|
||||||
|
|
||||||
|
@implementation FaceclawGraphics
|
||||||
|
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green {
|
||||||
|
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
|
||||||
|
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||||
|
const uint8_t *src = gray.bytes;
|
||||||
|
uint8_t *dst = rgba.mutableBytes;
|
||||||
|
uint8_t lut[256];
|
||||||
|
for (int i = 0; i < 256; i++) lut[i] = (uint8_t)round(pow(i / 255.0, 0.7) * 255.0);
|
||||||
|
for (NSInteger i = 0; i < width * height; i++) {
|
||||||
|
uint8_t value = lut[src[i]];
|
||||||
|
dst[i * 4] = green ? 0 : value;
|
||||||
|
dst[i * 4 + 1] = value;
|
||||||
|
dst[i * 4 + 2] = green ? 0 : value;
|
||||||
|
dst[i * 4 + 3] = 255;
|
||||||
|
}
|
||||||
|
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||||
|
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||||
|
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
|
||||||
|
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
|
||||||
|
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
|
||||||
|
if (image) CGImageRelease(image);
|
||||||
|
CGDataProviderRelease(provider);
|
||||||
|
CGColorSpaceRelease(space);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
|
||||||
|
if (size <= 0 || size > 1024) return nil;
|
||||||
|
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
|
||||||
|
if (!image) return nil;
|
||||||
|
image.size = CGSizeMake(size, size);
|
||||||
|
UIGraphicsImageRendererFormat *format = [UIGraphicsImageRendererFormat defaultFormat];
|
||||||
|
format.scale = 1;
|
||||||
|
format.opaque = NO;
|
||||||
|
UIGraphicsImageRenderer *renderer = [[UIGraphicsImageRenderer alloc] initWithSize:image.size format:format];
|
||||||
|
UIImage *raster = [renderer imageWithActions:^(UIGraphicsImageRendererContext *context) {
|
||||||
|
[image.CALayerTree renderInContext:context.CGContext];
|
||||||
|
}];
|
||||||
|
NSMutableData *rgba = [NSMutableData dataWithLength:size * size * 4];
|
||||||
|
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||||
|
CGContextRef context = CGBitmapContextCreate(rgba.mutableBytes, size, size, 8, size * 4, space,
|
||||||
|
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
|
||||||
|
CGContextDrawImage(context, CGRectMake(0, 0, size, size), raster.CGImage);
|
||||||
|
NSMutableData *gray = [NSMutableData dataWithLength:size * size];
|
||||||
|
const uint8_t *src = rgba.bytes;
|
||||||
|
uint8_t *dst = gray.mutableBytes;
|
||||||
|
for (NSInteger i = 0; i < size * size; i++) {
|
||||||
|
dst[i] = (uint8_t)round(0.2126 * src[i * 4] + 0.7152 * src[i * 4 + 1] + 0.0722 * src[i * 4 + 2]);
|
||||||
|
}
|
||||||
|
CGContextRelease(context);
|
||||||
|
CGColorSpaceRelease(space);
|
||||||
|
return gray;
|
||||||
|
}
|
||||||
|
@end
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
module FaceclawGraphics {
|
||||||
|
header "FaceclawGraphics.h"
|
||||||
|
export *
|
||||||
|
}
|
||||||
@@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly coordinator: MathCoordinator,
|
private readonly coordinator: MathCoordinator,
|
||||||
private readonly actions: LayerActions,
|
private readonly actions: LayerActions,
|
||||||
|
private readonly textInput?: () => Promise<void>,
|
||||||
) {
|
) {
|
||||||
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
||||||
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
|
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
|
||||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||||
}
|
}
|
||||||
|
|
||||||
private get mode(): CalculatorMode {
|
private get mode(): CalculatorMode {
|
||||||
@@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private refreshListening(): void {
|
private refreshListening(): void {
|
||||||
|
if (this.textInput) return;
|
||||||
if (!this.foreground) return;
|
if (!this.foreground) return;
|
||||||
if (this.listening === "continuous") {
|
if (this.listening === "continuous") {
|
||||||
if (!this.continuousActive) {
|
if (!this.continuousActive) {
|
||||||
@@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer {
|
|||||||
* that did not.
|
* that did not.
|
||||||
*/
|
*/
|
||||||
beginDictation(): void {
|
beginDictation(): void {
|
||||||
|
if (this.textInput) { void this.textInput(); return; }
|
||||||
if (this.dictating) return;
|
if (this.dictating) return;
|
||||||
this.dictating = true;
|
this.dictating = true;
|
||||||
void this.actions.startVoiceCapture(true);
|
void this.actions.startVoiceCapture(true);
|
||||||
@@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer {
|
|||||||
bodyBottom: number,
|
bodyBottom: number,
|
||||||
): void {
|
): void {
|
||||||
const prompt =
|
const prompt =
|
||||||
this.listening === "continuous"
|
this.textInput ? "Tap to type a problem on your phone"
|
||||||
|
: this.listening === "continuous"
|
||||||
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
|
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
|
||||||
: "Tap to say a problem — or use Voice input to type one";
|
: "Tap to say a problem — or use Voice input to type one";
|
||||||
const lines = wrapText(font, prompt, width - 96);
|
const lines = wrapText(font, prompt, width - 96);
|
||||||
@@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer {
|
|||||||
|
|
||||||
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
|
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||||
if (!this.lastInput) return;
|
if (!this.lastInput) return;
|
||||||
const label = "Heard";
|
const label = this.textInput ? "Entered" : "Heard";
|
||||||
image.drawText(font, 24, y, label, 110);
|
image.drawText(font, 24, y, label, 110);
|
||||||
const x = 24 + font.measureText(label) + 10;
|
const x = 24 + font.measureText(label) + 10;
|
||||||
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
|
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
|
||||||
@@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer {
|
|||||||
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
|
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
|
||||||
return gestureHints([
|
return gestureHints([
|
||||||
[GESTURE_SCROLL, scrollTarget],
|
[GESTURE_SCROLL, scrollTarget],
|
||||||
[GESTURE_CLICK, "mic"],
|
[GESTURE_CLICK, this.textInput ? "type" : "mic"],
|
||||||
[GESTURE_DOUBLE_CLICK, "back"],
|
[GESTURE_DOUBLE_CLICK, "back"],
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-46
@@ -1,47 +1,4 @@
|
|||||||
import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core'
|
import { Application } from '@nativescript/core'
|
||||||
import { bytesToHex, hexToBytes } from './util/hex-util'
|
import { createMainPage } from './phone-ui/main-page.ios'
|
||||||
|
|
||||||
// Milestone 1: exercise native UI, TypeScript, and an existing shared utility
|
Application.run({ create: createMainPage })
|
||||||
// without loading the Android services used by the full glasses application.
|
|
||||||
Application.run({
|
|
||||||
create: () => {
|
|
||||||
const page = new Page()
|
|
||||||
page.actionBarHidden = true
|
|
||||||
const scroll = new ScrollView()
|
|
||||||
const content = new StackLayout()
|
|
||||||
content.padding = 28
|
|
||||||
content.verticalAlignment = 'middle'
|
|
||||||
|
|
||||||
const label = (text: string, fontSize = 17) => {
|
|
||||||
const view = new Label()
|
|
||||||
view.text = text
|
|
||||||
view.fontSize = fontSize
|
|
||||||
view.textWrap = true
|
|
||||||
view.marginBottom = 20
|
|
||||||
content.addChild(view)
|
|
||||||
return view
|
|
||||||
}
|
|
||||||
|
|
||||||
label('Faceclaw', 36)
|
|
||||||
label('iOS port · First launch', 24)
|
|
||||||
label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`)
|
|
||||||
label('Glasses pairing and the Android features are still to be ported.')
|
|
||||||
const status = label('Tap below to test native input and shared TypeScript code.')
|
|
||||||
const button = new Button()
|
|
||||||
button.text = 'Run shared-code check'
|
|
||||||
button.accessibilityIdentifier = 'shared-code-check'
|
|
||||||
let checks = 0
|
|
||||||
button.on('tap', () => {
|
|
||||||
const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77'))
|
|
||||||
const passed = result === '46616365636C6177'
|
|
||||||
checks += 1
|
|
||||||
status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.`
|
|
||||||
console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`)
|
|
||||||
})
|
|
||||||
content.addChild(button)
|
|
||||||
scroll.content = content
|
|
||||||
page.content = scroll
|
|
||||||
page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded'))
|
|
||||||
return page
|
|
||||||
},
|
|
||||||
})
|
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
import { Dialogs, type ImageSource } from '@nativescript/core'
|
||||||
|
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||||
|
import { CalculatorLayer } from '../apps/calculator/calculator'
|
||||||
|
import { MathCoordinator } from '../apps/calculator/math/coordinator'
|
||||||
|
import { SurfaceCompositor } from '../graphics/surface-compositor'
|
||||||
|
import { flattenPlanes, type Plane } from '../graphics/plane'
|
||||||
|
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||||
|
import { previewPixels } from '../native/ios-graphics'
|
||||||
|
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
|
||||||
|
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||||
|
import { MenuLayer } from '../ui/menu'
|
||||||
|
import { createInProcessWindow } from '../ui/shell/in-process-window'
|
||||||
|
import { shell, type ShellWindow } from '../ui/shell/shell'
|
||||||
|
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
|
||||||
|
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||||
|
previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings'
|
||||||
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||||
|
|
||||||
|
/** Local display host: shared shell/windows, no device-service initialization. */
|
||||||
|
export class IosPreviewController {
|
||||||
|
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
|
||||||
|
private renderTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
|
private clockTimer: ReturnType<typeof setInterval> | null = null
|
||||||
|
private offSettings: (() => void) | null = null
|
||||||
|
private active = false
|
||||||
|
private shellDirty = true
|
||||||
|
private inputQueue: Promise<void> = Promise.resolve()
|
||||||
|
private lastLayout = ''
|
||||||
|
private prompting = false
|
||||||
|
private readonly actions: LayerActions = {
|
||||||
|
...noopLayerActions,
|
||||||
|
requestRender: () => this.requestShellRender(),
|
||||||
|
startTextSettingEdit: setting => this.editSetting(setting),
|
||||||
|
startTextSettingsEdit: async (settings, _title, finished) => {
|
||||||
|
for (const setting of settings) await this.editSetting(setting)
|
||||||
|
finished?.()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
|
||||||
|
private readonly onError: (message: string) => void) {
|
||||||
|
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||||
|
shell.configure({
|
||||||
|
actions: this.actions,
|
||||||
|
voiceInputEnabled: false,
|
||||||
|
getScreenTimeoutMs: () => null,
|
||||||
|
requestShellRender: () => this.requestShellRender(),
|
||||||
|
onWindowsChanged: () => this.requestShellRender(),
|
||||||
|
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
|
||||||
|
})
|
||||||
|
const launcher = createLauncherWindow({
|
||||||
|
actions: this.actions,
|
||||||
|
apps: () => [
|
||||||
|
{ appId: 'calculator', label: 'Calculator', icon: 'calculator' },
|
||||||
|
{ appId: 'display', label: 'Display', icon: 'settings' },
|
||||||
|
],
|
||||||
|
launchApp: id => this.launchApp(id),
|
||||||
|
uninstallApp: () => {},
|
||||||
|
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
|
||||||
|
setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() },
|
||||||
|
})
|
||||||
|
this.configureWindow(launcher)
|
||||||
|
shell.registerWindow(launcher)
|
||||||
|
shell.wake('window')
|
||||||
|
shell.focusWindow(launcher.windowId)
|
||||||
|
}
|
||||||
|
|
||||||
|
resume(): void {
|
||||||
|
if (this.active) return
|
||||||
|
this.active = true
|
||||||
|
this.offSettings = onAnySettingChanged(() => {
|
||||||
|
this.relayout()
|
||||||
|
shell.foregroundWindow()?.requestRender()
|
||||||
|
this.requestShellRender()
|
||||||
|
})
|
||||||
|
this.clockTimer = setInterval(() => this.requestShellRender(), 60_000)
|
||||||
|
this.relayout()
|
||||||
|
shell.foregroundWindow()?.requestRender()
|
||||||
|
this.requestShellRender()
|
||||||
|
console.log('[ios-preview] Main screen active')
|
||||||
|
}
|
||||||
|
pause(): void {
|
||||||
|
this.active = false
|
||||||
|
this.offSettings?.(); this.offSettings = null
|
||||||
|
if (this.clockTimer) clearInterval(this.clockTimer)
|
||||||
|
if (this.renderTimer) clearTimeout(this.renderTimer)
|
||||||
|
this.clockTimer = this.renderTimer = null
|
||||||
|
}
|
||||||
|
private configureWindow(window: ShellWindow): void {
|
||||||
|
this.compositor.configureSurface(window.surfaceId, {
|
||||||
|
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
|
||||||
|
})
|
||||||
|
this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
|
||||||
|
}
|
||||||
|
private relayout(): void {
|
||||||
|
const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}`
|
||||||
|
if (layout === this.lastLayout) return
|
||||||
|
this.lastLayout = layout
|
||||||
|
for (const window of shell.getWindows()) {
|
||||||
|
this.configureWindow(window)
|
||||||
|
window.relayout?.()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||||
|
const image = flattenPlanes(planes)
|
||||||
|
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
|
||||||
|
this.scheduleFrame()
|
||||||
|
}
|
||||||
|
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
|
||||||
|
private scheduleFrame(): void {
|
||||||
|
if (!this.active || this.renderTimer !== null) return
|
||||||
|
// Coalesce window/chrome updates into one preview, capped at 30 fps.
|
||||||
|
this.renderTimer = setTimeout(() => {
|
||||||
|
this.renderTimer = null
|
||||||
|
try {
|
||||||
|
if (this.shellDirty) {
|
||||||
|
this.shellDirty = false
|
||||||
|
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
|
||||||
|
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
|
||||||
|
this.compositor.setUnderlayDim(1, shell.underlayDim())
|
||||||
|
}
|
||||||
|
const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green')
|
||||||
|
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||||
|
} catch (error) { this.fail(error) }
|
||||||
|
}, 33)
|
||||||
|
}
|
||||||
|
private fail(error: unknown): void {
|
||||||
|
const message = error instanceof Error ? error.stack ?? error.message : String(error)
|
||||||
|
console.error(`[ios-preview] ${message}`)
|
||||||
|
this.onError(error instanceof Error ? error.message : String(error))
|
||||||
|
}
|
||||||
|
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
|
||||||
|
this.inputQueue = this.inputQueue.then(async () => {
|
||||||
|
if (!this.active) return
|
||||||
|
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
|
||||||
|
await this.mirrorTap(nx, ny)
|
||||||
|
} else {
|
||||||
|
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
|
||||||
|
if (origin === 'ring' && type.startsWith('swipe-')) {
|
||||||
|
if (type === 'swipe-left' || type === 'swipe-right') return
|
||||||
|
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
|
||||||
|
}
|
||||||
|
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
|
||||||
|
await shell.receiveInput(event)
|
||||||
|
this.requestShellRender()
|
||||||
|
}
|
||||||
|
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
|
||||||
|
}).catch(error => this.fail(error))
|
||||||
|
}
|
||||||
|
private async mirrorTap(nx: number, ny: number): Promise<void> {
|
||||||
|
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||||
|
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||||
|
const window = shell.foregroundWindow()
|
||||||
|
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) {
|
||||||
|
const target = shell.windowAtSidebarPoint(x, y)
|
||||||
|
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (window && !shell.hasOverlay()) {
|
||||||
|
const rect = appViewportRect(window.heightMode, window.appId)
|
||||||
|
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
|
||||||
|
shell.focusWindow(window.windowId)
|
||||||
|
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
|
||||||
|
}
|
||||||
|
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
|
||||||
|
this.requestShellRender()
|
||||||
|
}
|
||||||
|
launchApp(id: string): void {
|
||||||
|
const existing = shell.getWindows().find(window => window.appId === id)
|
||||||
|
if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return }
|
||||||
|
const surfaceId = `window:${id}`
|
||||||
|
let render = () => {}
|
||||||
|
const actions = { ...this.actions, requestRender: () => render() }
|
||||||
|
const layer = id === 'calculator'
|
||||||
|
? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp())
|
||||||
|
: new MenuLayer('Display', [
|
||||||
|
{ label: 'Change screen size', onSelect: () => this.cycleDisplayMode() },
|
||||||
|
{ label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } },
|
||||||
|
{ label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } },
|
||||||
|
], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true })
|
||||||
|
const app = createInProcessWindow({
|
||||||
|
appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display',
|
||||||
|
iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings',
|
||||||
|
closeable: true, actions, baseLayer: layer,
|
||||||
|
receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined,
|
||||||
|
submitFrame: planes => this.submit(surfaceId, planes),
|
||||||
|
setSurfaceVisible: visible => {
|
||||||
|
if (layer instanceof CalculatorLayer) layer.setForeground(visible)
|
||||||
|
this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame()
|
||||||
|
},
|
||||||
|
removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() },
|
||||||
|
onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() },
|
||||||
|
})
|
||||||
|
render = app.requestRender
|
||||||
|
if (layer instanceof CalculatorLayer) layer.requestRender = render
|
||||||
|
this.configureWindow(app.window)
|
||||||
|
shell.registerWindow(app.window)
|
||||||
|
shell.focusWindow(app.window.windowId)
|
||||||
|
app.requestRender(); this.requestShellRender()
|
||||||
|
}
|
||||||
|
cycleDisplayMode(): void {
|
||||||
|
const values = DISPLAY_MODE_VALUES
|
||||||
|
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
|
||||||
|
}
|
||||||
|
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
|
||||||
|
async typeIntoApp(): Promise<void> {
|
||||||
|
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
|
||||||
|
this.prompting = true
|
||||||
|
try {
|
||||||
|
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
|
||||||
|
message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
|
||||||
|
if (result.result) shell.sendTextToForegroundWindow(result.text)
|
||||||
|
} catch (error) { this.fail(error) }
|
||||||
|
finally { this.prompting = false }
|
||||||
|
}
|
||||||
|
private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise<void> {
|
||||||
|
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' })
|
||||||
|
if (result.result) setting.set(result.text)
|
||||||
|
}
|
||||||
|
}
|
||||||
+7
-17
@@ -1,12 +1,10 @@
|
|||||||
import { GrayImage } from "./image";
|
import { GrayImage } from "./image";
|
||||||
|
import { rasterizeSvg } from "../native/svg-rasterizer";
|
||||||
declare const com: any;
|
|
||||||
declare const global: any;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Vector icons for window indicators (and anywhere else). SVGs are rendered
|
* Vector icons for window indicators (and anywhere else). SVGs are rendered
|
||||||
* once to a correctly sized grayscale bitmap by the Java IconRenderer (a
|
* once to a correctly sized grayscale bitmap by the platform rasterizer
|
||||||
* small SVG subset: path/circle/rect/line/polyline) and cached here. To add
|
* (Android IconRenderer or iOS SVGKit) and cached here. To add
|
||||||
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
|
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
|
||||||
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
|
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
|
||||||
* Noun Project glyphs also work.
|
* Noun Project glyphs also work.
|
||||||
@@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag
|
|||||||
if (cached !== undefined) return cached;
|
if (cached !== undefined) return cached;
|
||||||
|
|
||||||
let icon: GrayImage | null = null;
|
let icon: GrayImage | null = null;
|
||||||
if (global.isAndroid) {
|
try {
|
||||||
try {
|
icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH);
|
||||||
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH);
|
} catch (error) {
|
||||||
if (bytes && bytes.length >= size * size) {
|
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
||||||
icon = new GrayImage(size, size, 0);
|
|
||||||
for (let i = 0; i < size * size; i++) {
|
|
||||||
icon.pixels[i] = bytes[i] & 0xff;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
cache.set(key, icon);
|
cache.set(key, icon);
|
||||||
return icon;
|
return icon;
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
/** Platform-independent 8bpp surface composition for the local display. */
|
||||||
|
export type SurfaceRect = { x: number; y: number; width: number; height: number }
|
||||||
|
export type SurfaceConfiguration = SurfaceRect & {
|
||||||
|
zOrder: number
|
||||||
|
transparency: 'opaque' | 'color-key'
|
||||||
|
}
|
||||||
|
type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean }
|
||||||
|
|
||||||
|
export class SurfaceCompositor {
|
||||||
|
private readonly surfaces = new Map<string, Surface>()
|
||||||
|
private dimBelow = 0
|
||||||
|
private dimFactor = 1
|
||||||
|
private blanked = false
|
||||||
|
constructor(readonly width: number, readonly height: number) {
|
||||||
|
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
|
||||||
|
throw new Error('Invalid compositor size')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
configureSurface(id: string, options: SurfaceConfiguration): void {
|
||||||
|
if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger)
|
||||||
|
|| options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry')
|
||||||
|
const previous = this.surfaces.get(id)
|
||||||
|
const pixels = previous?.width === options.width && previous.height === options.height
|
||||||
|
? previous.pixels : new Uint8Array(options.width * options.height)
|
||||||
|
this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true })
|
||||||
|
}
|
||||||
|
removeSurface(id: string): void { this.surfaces.delete(id) }
|
||||||
|
setSurfaceVisible(id: string, visible: boolean): void {
|
||||||
|
const surface = this.surfaces.get(id)
|
||||||
|
if (surface) surface.visible = visible
|
||||||
|
}
|
||||||
|
setUnderlayDim(belowZOrder: number, factor: number): void {
|
||||||
|
if (!Number.isFinite(factor)) throw new Error('Invalid dim factor')
|
||||||
|
this.dimBelow = belowZOrder
|
||||||
|
this.dimFactor = Math.max(0, Math.min(1, factor))
|
||||||
|
}
|
||||||
|
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
|
||||||
|
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
|
||||||
|
const surface = this.surfaces.get(id)
|
||||||
|
if (!surface) throw new Error(`Unknown surface: ${id}`)
|
||||||
|
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|
||||||
|
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) {
|
||||||
|
throw new Error('Invalid frame buffer or rectangle')
|
||||||
|
}
|
||||||
|
const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width)
|
||||||
|
const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height)
|
||||||
|
for (let y = top; y < bottom; y++) {
|
||||||
|
if (right <= left) break
|
||||||
|
const source = (y - rect.y) * rect.width + left - rect.x
|
||||||
|
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
composite(): Uint8Array {
|
||||||
|
const output = new Uint8Array(this.width * this.height)
|
||||||
|
if (this.blanked) return output
|
||||||
|
const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder)
|
||||||
|
for (const surface of surfaces) {
|
||||||
|
const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width)
|
||||||
|
const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height)
|
||||||
|
const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1
|
||||||
|
for (let y = top; y < bottom; y++) {
|
||||||
|
let source = (y - surface.y) * surface.width + left - surface.x
|
||||||
|
let target = y * this.width + left
|
||||||
|
for (let x = left; x < right; x++, source++, target++) {
|
||||||
|
const value = surface.pixels[source]
|
||||||
|
// Test the original value: opaque near-black must still cover the
|
||||||
|
// window beneath, even when dimming rounds it down to black.
|
||||||
|
if (surface.transparency === 'color-key' && value === 0) continue
|
||||||
|
output[target] = Math.round(value * dim)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import { ImageSource } from '@nativescript/core'
|
||||||
|
import { GrayImage } from '../graphics/image'
|
||||||
|
|
||||||
|
declare const FaceclawGraphics: any
|
||||||
|
|
||||||
|
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
|
||||||
|
// NSData copies the buffer; the UIImage owns its pixels after this call.
|
||||||
|
const copy = new Uint8Array(pixels)
|
||||||
|
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||||
|
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
|
||||||
|
if (!native) throw new Error('Could not create preview image')
|
||||||
|
return new ImageSource(native)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function rasterizeSvg(svg: string, size: number): GrayImage | null {
|
||||||
|
const data = FaceclawGraphics.renderSVGSize(svg, size)
|
||||||
|
if (!data) return null
|
||||||
|
const image = new GrayImage(size, size, 0)
|
||||||
|
image.pixels.set(new Uint8Array(interop.bufferFromData(data)))
|
||||||
|
return image
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import { ApplicationSettings } from '@nativescript/core'
|
||||||
|
|
||||||
|
// Main-isolate settings for the initial iOS port. Worker synchronization will
|
||||||
|
// be added when worker-hosted apps are enabled on iOS.
|
||||||
|
const listeners = new Set<(key: string) => void>()
|
||||||
|
function changed(key: string): void {
|
||||||
|
setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0)
|
||||||
|
}
|
||||||
|
export function getStringSetting(key: string, fallback: string): string {
|
||||||
|
return ApplicationSettings.getString(key, fallback)
|
||||||
|
}
|
||||||
|
export function setStringSetting(key: string, value: string): void {
|
||||||
|
if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return
|
||||||
|
ApplicationSettings.setString(key, value)
|
||||||
|
changed(key)
|
||||||
|
}
|
||||||
|
export function getBooleanSetting(key: string, fallback: boolean): boolean {
|
||||||
|
return ApplicationSettings.getBoolean(key, fallback)
|
||||||
|
}
|
||||||
|
export function setBooleanSetting(key: string, value: boolean): void {
|
||||||
|
if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return
|
||||||
|
ApplicationSettings.setBoolean(key, value)
|
||||||
|
changed(key)
|
||||||
|
}
|
||||||
|
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
|
||||||
|
listeners.add(listener)
|
||||||
|
return () => { listeners.delete(listener) }
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { rasterizeSvg as render } from './ios-graphics'
|
||||||
|
export function rasterizeSvg(svg: string, size: number, strokeWidth: number) {
|
||||||
|
// Shared icon sources rely on the Android renderer's white, rounded
|
||||||
|
// stroke defaults. Supply those explicitly for a standards SVG renderer.
|
||||||
|
const source = svg.replace(/<svg\b([^>]*)>/, (_tag, attributes: string) => {
|
||||||
|
const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24',
|
||||||
|
stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth),
|
||||||
|
'stroke-linecap': 'round', 'stroke-linejoin': 'round' }
|
||||||
|
const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes))
|
||||||
|
.map(([key, value]) => ` ${key}="${value}"`).join('')
|
||||||
|
return `<svg${attributes}${missing}>`
|
||||||
|
})
|
||||||
|
return render(source, size)
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import { GrayImage } from '../graphics/image'
|
||||||
|
declare const com: any
|
||||||
|
export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null {
|
||||||
|
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth)
|
||||||
|
if (!bytes || bytes.length < size * size) return null
|
||||||
|
const image = new GrayImage(size, size, 0)
|
||||||
|
for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff
|
||||||
|
return image
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core'
|
||||||
|
import { IosPreviewController } from '../g2/ios-preview-controller'
|
||||||
|
import { PhoneGestureRecognizer } from './phone-gestures'
|
||||||
|
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
|
||||||
|
|
||||||
|
export function createMainPage(): Page {
|
||||||
|
const page = new Page()
|
||||||
|
page.actionBarHidden = true
|
||||||
|
const root = new GridLayout()
|
||||||
|
root.rows = 'auto,*'
|
||||||
|
page.content = root
|
||||||
|
const header = new GridLayout()
|
||||||
|
header.columns = '*,auto'; header.padding = '12 20'
|
||||||
|
const title = new Label()
|
||||||
|
title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold'
|
||||||
|
header.addChild(title)
|
||||||
|
const state = new Label()
|
||||||
|
state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle'
|
||||||
|
GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header)
|
||||||
|
const body = new GridLayout()
|
||||||
|
GridLayout.setRow(body, 1); root.addChild(body)
|
||||||
|
const mirror = new Image()
|
||||||
|
mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black')
|
||||||
|
mirror.accessibilityIdentifier = 'glasses-preview'
|
||||||
|
mirror.accessibilityLabel = 'Interactive glasses screen preview'
|
||||||
|
mirror.verticalAlignment = 'middle'; body.addChild(mirror)
|
||||||
|
const controls = new GridLayout()
|
||||||
|
controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls)
|
||||||
|
const tabs = new GridLayout()
|
||||||
|
tabs.columns = '*,*,*'; controls.addChild(tabs)
|
||||||
|
const pad = new GridLayout()
|
||||||
|
pad.borderRadius = 18; pad.backgroundColor = new Color('#202522')
|
||||||
|
pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center'
|
||||||
|
pad.accessibilityIdentifier = 'gesture-pad'
|
||||||
|
GridLayout.setRow(pad, 1); controls.addChild(pad)
|
||||||
|
const focus = new Label()
|
||||||
|
focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true
|
||||||
|
focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle'
|
||||||
|
focus.isUserInteractionEnabled = false; pad.addChild(focus)
|
||||||
|
const legend = new Label()
|
||||||
|
legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center'
|
||||||
|
legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom'
|
||||||
|
legend.isUserInteractionEnabled = false; pad.addChild(legend)
|
||||||
|
const settings = new StackLayout()
|
||||||
|
settings.verticalAlignment = 'middle'
|
||||||
|
GridLayout.setRow(settings, 1); controls.addChild(settings)
|
||||||
|
const errorLabel = new Label()
|
||||||
|
errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse'
|
||||||
|
settings.addChild(errorLabel)
|
||||||
|
let selectedTab: 'watch' | 'ring' | 'settings' = 'watch'
|
||||||
|
const controller = new IosPreviewController((image, label) => {
|
||||||
|
mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label
|
||||||
|
}, message => {
|
||||||
|
errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings')
|
||||||
|
})
|
||||||
|
const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch'))
|
||||||
|
const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => {
|
||||||
|
const size = mirror.getActualSize()
|
||||||
|
if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height)
|
||||||
|
})
|
||||||
|
const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => {
|
||||||
|
if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return
|
||||||
|
recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() })
|
||||||
|
}
|
||||||
|
pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput))
|
||||||
|
function button(text: string, action: () => void): Button {
|
||||||
|
const view = new Button()
|
||||||
|
view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action)
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
const tabButtons: Button[] = []
|
||||||
|
function selectTab(tab: typeof selectedTab): void {
|
||||||
|
padInput.cancel(); selectedTab = tab
|
||||||
|
pad.visibility = tab === 'settings' ? 'collapse' : 'visible'
|
||||||
|
settings.visibility = tab === 'settings' ? 'visible' : 'collapse'
|
||||||
|
for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5
|
||||||
|
focus.text = tab === 'ring' ? 'Ring' : 'Watch controls'
|
||||||
|
legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu'
|
||||||
|
layout()
|
||||||
|
}
|
||||||
|
for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) {
|
||||||
|
const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab))
|
||||||
|
GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view)
|
||||||
|
}
|
||||||
|
const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode())
|
||||||
|
settings.addChild(sizeButton)
|
||||||
|
settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green')))
|
||||||
|
const help = new Label()
|
||||||
|
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
|
||||||
|
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
|
||||||
|
const footer = new GridLayout()
|
||||||
|
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
|
||||||
|
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
|
||||||
|
const menu = button('Menu', () => {
|
||||||
|
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
|
||||||
|
})
|
||||||
|
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
|
||||||
|
for (const [index, view] of [back, menu, keyboard].entries()) {
|
||||||
|
view.padding = '16 8'; view.fontSize = 14
|
||||||
|
GridLayout.setColumn(view, index); footer.addChild(view)
|
||||||
|
}
|
||||||
|
let lastLayout = ''
|
||||||
|
function layout(): void {
|
||||||
|
const size = root.getActualSize()
|
||||||
|
if (size.width <= 0 || size.height <= 0) return
|
||||||
|
const key = `${size.width}:${size.height}:${selectedTab}`
|
||||||
|
if (key === lastLayout) return
|
||||||
|
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
|
||||||
|
const landscape = size.width > size.height
|
||||||
|
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
|
||||||
|
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
|
||||||
|
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
|
||||||
|
mirror.width = width; mirror.height = width * 3 / 4
|
||||||
|
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
|
||||||
|
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
|
||||||
|
}
|
||||||
|
// iOS emits layoutChanged inside its layout pass. Defer mutations so
|
||||||
|
// NativeScript does not clear our new layout request at the end of that pass.
|
||||||
|
let layoutTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
|
root.on('layoutChanged', () => {
|
||||||
|
if (layoutTimer !== null) return
|
||||||
|
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
|
||||||
|
})
|
||||||
|
const pause = () => {
|
||||||
|
if (layoutTimer !== null) clearTimeout(layoutTimer)
|
||||||
|
layoutTimer = null
|
||||||
|
padInput.cancel(); mirrorInput.cancel(); controller.pause()
|
||||||
|
}
|
||||||
|
const resume = () => controller.resume()
|
||||||
|
let offSettings: (() => void) | null = null
|
||||||
|
page.on('loaded', () => {
|
||||||
|
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
|
||||||
|
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
|
||||||
|
selectTab(selectedTab); controller.resume()
|
||||||
|
})
|
||||||
|
page.on('unloaded', () => {
|
||||||
|
pause(); offSettings?.(); offSettings = null
|
||||||
|
Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume)
|
||||||
|
})
|
||||||
|
return page
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */
|
||||||
|
declare function setTimeout(callback: () => void, ms: number): number
|
||||||
|
declare function clearTimeout(id: number): void
|
||||||
|
export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release'
|
||||||
|
| 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right'
|
||||||
|
export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number }
|
||||||
|
export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void }
|
||||||
|
const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType<typeof setTimeout>) }
|
||||||
|
|
||||||
|
export class PhoneGestureRecognizer {
|
||||||
|
private start: PhoneTouch | null = null
|
||||||
|
private hold: unknown = null
|
||||||
|
private tap: unknown = null
|
||||||
|
private lastTap: { x: number; y: number } | null = null
|
||||||
|
private holding = false
|
||||||
|
private moved = false
|
||||||
|
private multi = false
|
||||||
|
private secondTap = false
|
||||||
|
constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void,
|
||||||
|
private readonly timer: GestureClock = clock) {}
|
||||||
|
|
||||||
|
touch(event: PhoneTouch): void {
|
||||||
|
if (event.action === 'cancel') { this.cancel(); return }
|
||||||
|
if (event.action === 'down' && !this.start) {
|
||||||
|
this.start = event
|
||||||
|
this.holding = this.moved = this.multi = false
|
||||||
|
this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36
|
||||||
|
if (this.secondTap) { this.clearTap() }
|
||||||
|
else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) }
|
||||||
|
this.hold = this.timer.set(() => {
|
||||||
|
this.hold = null
|
||||||
|
if (!this.start || this.moved || this.multi) return
|
||||||
|
this.holding = true
|
||||||
|
this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y)
|
||||||
|
}, 500)
|
||||||
|
}
|
||||||
|
if (!this.start) return
|
||||||
|
if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() }
|
||||||
|
const dx = event.x - this.start.x, dy = event.y - this.start.y
|
||||||
|
if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() }
|
||||||
|
if (event.action !== 'up') return
|
||||||
|
// Some platforms emit an up for each finger. Complete the multi-touch
|
||||||
|
// gesture only once, and suppress the trailing one-finger up events.
|
||||||
|
this.clearHold()
|
||||||
|
const start = this.start
|
||||||
|
this.start = null
|
||||||
|
if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return }
|
||||||
|
if (this.multi) { this.emit('double-tap', start.x, start.y); return }
|
||||||
|
if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) {
|
||||||
|
this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y)
|
||||||
|
} else if (!this.moved) {
|
||||||
|
if (this.secondTap) this.emit('double-tap', start.x, start.y)
|
||||||
|
else {
|
||||||
|
this.lastTap = { x: start.x, y: start.y }
|
||||||
|
this.tap = this.timer.set(() => {
|
||||||
|
const tap = this.lastTap
|
||||||
|
this.tap = null; this.lastTap = null
|
||||||
|
if (tap) this.emit('tap', tap.x, tap.y)
|
||||||
|
}, 280)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancel(): void {
|
||||||
|
this.clearHold(); this.clearTap()
|
||||||
|
if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y)
|
||||||
|
this.start = null; this.holding = false
|
||||||
|
}
|
||||||
|
private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null }
|
||||||
|
private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null }
|
||||||
|
}
|
||||||
@@ -144,6 +144,8 @@ export type ShellWindow = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export type ShellConfig = {
|
export type ShellConfig = {
|
||||||
|
/** Hosts without microphone support hide and reject voice entry points. */
|
||||||
|
voiceInputEnabled?: boolean;
|
||||||
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
|
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
|
||||||
actions: LayerActions;
|
actions: LayerActions;
|
||||||
getScreenTimeoutMs: () => number | null;
|
getScreenTimeoutMs: () => number | null;
|
||||||
@@ -901,7 +903,7 @@ class Shell {
|
|||||||
handsFree?: boolean;
|
handsFree?: boolean;
|
||||||
defaultTarget: "assistant" | "app";
|
defaultTarget: "assistant" | "app";
|
||||||
}): void {
|
}): void {
|
||||||
if (this.voiceDialogPending) return;
|
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
|
||||||
this.voiceDialogPending = true;
|
this.voiceDialogPending = true;
|
||||||
void (async () => {
|
void (async () => {
|
||||||
let ready = true;
|
let ready = true;
|
||||||
@@ -1005,6 +1007,7 @@ class Shell {
|
|||||||
* the dialog finishes on click instead of long-press-release.
|
* the dialog finishes on click instead of long-press-release.
|
||||||
*/
|
*/
|
||||||
startVoiceInput(): void {
|
startVoiceInput(): void {
|
||||||
|
if (this.config.voiceInputEnabled === false) return;
|
||||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
||||||
// The transcript is aimed at the window whose menu requested it; the menu
|
// The transcript is aimed at the window whose menu requested it; the menu
|
||||||
// entry point defaults the highlight to Type Into App.
|
// entry point defaults the highlight to Type Into App.
|
||||||
@@ -1334,7 +1337,7 @@ class Shell {
|
|||||||
ctx.stack.pop();
|
ctx.stack.pop();
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
...(this.config.voiceInputEnabled === false ? [] : [{
|
||||||
label: "Voice input",
|
label: "Voice input",
|
||||||
onSelect: (ctx) => {
|
onSelect: (ctx) => {
|
||||||
// The transcript is aimed at the foreground window, so keep focus
|
// The transcript is aimed at the foreground window, so keep focus
|
||||||
@@ -1343,7 +1346,7 @@ class Shell {
|
|||||||
ctx.stack.pop();
|
ctx.stack.pop();
|
||||||
this.startVoiceInput();
|
this.startVoiceInput();
|
||||||
},
|
},
|
||||||
},
|
}]),
|
||||||
);
|
);
|
||||||
items.push({
|
items.push({
|
||||||
label: "Debug",
|
label: "Debug",
|
||||||
|
|||||||
Executable
+17
@@ -0,0 +1,17 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
# Prefer Homebrew Ruby over the older macOS system Ruby when available.
|
||||||
|
for ruby_bin in /opt/homebrew/opt/ruby/bin /usr/local/opt/ruby/bin; do
|
||||||
|
if [[ -x "$ruby_bin/ruby" ]]; then
|
||||||
|
export PATH="$ruby_bin:$PATH"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if command -v ruby >/dev/null 2>&1; then
|
||||||
|
gem_bin="$(ruby -e 'puts Gem.bindir')"
|
||||||
|
export PATH="$gem_bin:$PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec ns "$@"
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { loader } = require('./helpers/load-typescript.cjs');
|
||||||
|
|
||||||
|
function harness(provider = 'elevenlabs', continuous = true) {
|
||||||
|
const timers = new Map();
|
||||||
|
let timerId = 0;
|
||||||
|
const sockets = [];
|
||||||
|
const events = [], statuses = [], errors = [];
|
||||||
|
const load = loader({
|
||||||
|
setTimeout(fn, delay) { const id = ++timerId; timers.set(id, { fn, delay }); return id; },
|
||||||
|
clearTimeout(id) { timers.delete(id); },
|
||||||
|
console: { log() {}, warn() {} },
|
||||||
|
com: { faceclaw: { app: {
|
||||||
|
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
|
||||||
|
FaceclawWebSocket: class {
|
||||||
|
constructor(url, listener) { Object.assign(this, { url, listener, sent: [], closed: false, reject: false }); sockets.push(this); }
|
||||||
|
sendText(text) { if (this.reject) return false; this.sent.push(text); return true; }
|
||||||
|
sendBinary(bytes) { if (this.reject) return false; this.sent.push(bytes); return true; }
|
||||||
|
close() { this.closed = true; }
|
||||||
|
}, } }, },
|
||||||
|
}, {
|
||||||
|
'./cloud-stt': {
|
||||||
|
CLOUD_STT_SAMPLE_RATE: 16000,
|
||||||
|
encodeBase64: bytes => Buffer.from(bytes).toString('base64'),
|
||||||
|
toJavaBytes: bytes => bytes,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const names = { elevenlabs: 'ElevenLabsSttClient', openai: 'OpenAiRealtimeSttClient', soniox: 'SonioxSttClient' };
|
||||||
|
const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]];
|
||||||
|
const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts');
|
||||||
|
const client = new ReconnectingSttClient(options => new Provider(options), {
|
||||||
|
apiKey: 'test', onTranscript: event => events.push(event),
|
||||||
|
onStatus: status => statuses.push(status), onError: error => errors.push(error),
|
||||||
|
}, () => continuous);
|
||||||
|
client.start();
|
||||||
|
return { client, sockets, events, statuses, errors, timers,
|
||||||
|
open() { sockets.at(-1).listener.onOpen(); },
|
||||||
|
message(message, socket = sockets.at(-1)) { socket.listener.onTextMessage(JSON.stringify(message)); },
|
||||||
|
tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); return timer.delay; },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pcm(amplitude, ms = 50) {
|
||||||
|
const bytes = new Uint8Array(ms * 32);
|
||||||
|
for (let i = 0; i < bytes.length; i += 2) { bytes[i] = amplitude & 255; bytes[i + 1] = amplitude >> 8; }
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const provider of ['elevenlabs', 'openai', 'soniox']) {
|
||||||
|
test(`${provider}: failure and close reconnect, stale callbacks are ignored, stop cancels retry`, () => {
|
||||||
|
const h = harness(provider);
|
||||||
|
h.open();
|
||||||
|
const old = h.sockets[0];
|
||||||
|
old.listener.onFailure('network unavailable');
|
||||||
|
old.listener.onClosed(); // Both callbacks must schedule only one retry.
|
||||||
|
assert.equal(h.timers.size, 1);
|
||||||
|
assert.equal(old.closed, true);
|
||||||
|
h.client.acceptPcm(pcm(2000));
|
||||||
|
assert.equal(h.tick(), 1000);
|
||||||
|
assert.equal(h.sockets.length, 2);
|
||||||
|
h.open();
|
||||||
|
assert.ok(h.sockets[1].sent.length > 0);
|
||||||
|
const eventCount = h.events.length;
|
||||||
|
old.listener.onTextMessage(JSON.stringify({ message_type: 'partial_transcript', text: 'stale' }));
|
||||||
|
old.listener.onOpen();
|
||||||
|
assert.equal(h.events.length, eventCount);
|
||||||
|
h.sockets[1].listener.onClosed();
|
||||||
|
assert.equal(h.timers.size, 1);
|
||||||
|
h.client.stop();
|
||||||
|
assert.equal(h.timers.size, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test(`${provider}: rejected audio send triggers recovery and requeues the packet`, () => {
|
||||||
|
const h = harness(provider);
|
||||||
|
h.open();
|
||||||
|
h.sockets[0].reject = true;
|
||||||
|
h.client.acceptPcm(pcm(2000));
|
||||||
|
assert.equal(h.timers.size, 1);
|
||||||
|
h.tick(); h.open();
|
||||||
|
assert.ok(h.sockets[1].sent.length > (provider === 'elevenlabs' ? 0 : 1));
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test(`${provider}: auth failure stops instead of retrying indefinitely`, () => {
|
||||||
|
const h = harness(provider);
|
||||||
|
h.sockets[0].listener.onFailure("Expected HTTP 101 response but was '401 Unauthorized'");
|
||||||
|
assert.equal(h.timers.size, 0);
|
||||||
|
assert.equal(h.errors.length, 1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test('reconnect preserves visible partials once and does not replay already-sent audio', () => {
|
||||||
|
const h = harness(); h.open();
|
||||||
|
h.client.acceptPcm(pcm(2000));
|
||||||
|
h.message({ message_type: 'partial_transcript', text: 'Before outage' });
|
||||||
|
h.sockets[0].listener.onFailure('offline');
|
||||||
|
assert.equal(h.events.at(-1).text, 'Before outage');
|
||||||
|
assert.equal(h.events.at(-1).isFinal, true);
|
||||||
|
h.tick(); h.open();
|
||||||
|
assert.equal(h.sockets[1].sent.length, 0);
|
||||||
|
h.message({ message_type: 'partial_transcript', text: 'After outage' });
|
||||||
|
assert.equal(h.events.at(-1).text, 'After outage');
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('pending audio is bounded to ten seconds and backoff is capped', () => {
|
||||||
|
const h = harness();
|
||||||
|
for (let i = 0; i < 20; i++) h.client.acceptPcm(pcm(i, 1000));
|
||||||
|
h.open();
|
||||||
|
const sentBytes = h.sockets[0].sent.reduce((sum, text) => sum + Buffer.from(JSON.parse(text).audio_base_64, 'base64').length, 0);
|
||||||
|
assert.equal(sentBytes, 320000);
|
||||||
|
for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {
|
||||||
|
h.sockets.at(-1).listener.onFailure('offline');
|
||||||
|
assert.equal(h.tick(), delay);
|
||||||
|
}
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('finishing while disconnected cancels retries; finishing on an open socket times out cleanly', () => {
|
||||||
|
const h = harness(); h.open();
|
||||||
|
h.sockets[0].listener.onClosed();
|
||||||
|
h.client.finish();
|
||||||
|
assert.equal(h.timers.size, 0);
|
||||||
|
const live = harness(); live.open();
|
||||||
|
live.client.finish();
|
||||||
|
assert.equal(live.tick(), 5000);
|
||||||
|
assert.equal(live.sockets[0].closed, true);
|
||||||
|
live.sockets[0].listener.onFailure('late');
|
||||||
|
assert.equal(live.timers.size, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('only continuous capture commits at pauses, keeping ElevenLabs connected', () => {
|
||||||
|
for (const continuous of [false, true]) {
|
||||||
|
const h = harness('elevenlabs', continuous); h.open();
|
||||||
|
h.client.acceptPcm(pcm(3000));
|
||||||
|
h.client.acceptPcm(pcm(0, 1500));
|
||||||
|
const commits = h.sockets[0].sent.map(JSON.parse).filter(m => m.commit);
|
||||||
|
assert.equal(commits.length, continuous ? 1 : 0);
|
||||||
|
if (continuous) {
|
||||||
|
h.message({ message_type: 'committed_transcript', text: 'First.' });
|
||||||
|
h.message({ message_type: 'committed_transcript_with_timestamps', text: 'First.' });
|
||||||
|
assert.equal(h.events.length, 1);
|
||||||
|
assert.equal(h.events[0].paragraphBreakAfter, true);
|
||||||
|
assert.equal(h.sockets[0].closed, false);
|
||||||
|
}
|
||||||
|
h.client.stop();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('OpenAI keeps successive items separate and drains out-of-order completions in audio order', () => {
|
||||||
|
const h = harness('openai'); h.open();
|
||||||
|
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
|
||||||
|
h.message({ type: 'input_audio_buffer.committed', item_id: 'a' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'a', delta: 'Fir' });
|
||||||
|
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
|
||||||
|
h.message({ type: 'input_audio_buffer.committed', item_id: 'b' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'Second.' });
|
||||||
|
assert.equal(h.events.some(e => e.isFinal), false);
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'First.' });
|
||||||
|
assert.deepEqual(h.events.filter(e => e.isFinal).map(e => e.text), ['First.', 'Second.']);
|
||||||
|
assert.equal(h.sockets[0].closed, false);
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'c', delta: 'Third.' });
|
||||||
|
assert.equal(h.events.at(-1).text, 'Third.');
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Soniox speaker and pause formatting remains separate from dictation text and resets on finalize', () => {
|
||||||
|
const h = harness('soniox'); h.open();
|
||||||
|
h.message({ tokens: [
|
||||||
|
{ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1', is_final: true },
|
||||||
|
{ text: ' Hi.', start_ms: 550, end_ms: 800, speaker: '2', is_final: false },
|
||||||
|
] });
|
||||||
|
assert.equal(h.events.at(-1).text, 'Hello. Hi.');
|
||||||
|
assert.equal(h.events.at(-1).transcribeText, 'Hello.\nHi.');
|
||||||
|
h.message({ tokens: [
|
||||||
|
{ text: ' Yes.', start_ms: 550, end_ms: 800, speaker: '2', is_final: true },
|
||||||
|
{ text: '<fin>', is_final: true },
|
||||||
|
] });
|
||||||
|
assert.equal(h.events.find(e => e.isFinal).transcribeText, 'Hello.\nYes.');
|
||||||
|
h.message({ tokens: [{ text: 'New.', start_ms: 3000, end_ms: 3400, speaker: '2', is_final: false }] });
|
||||||
|
assert.equal(h.events.at(-1).text, 'New.');
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an unsolicited Soniox session end finalizes visible text and reconnects', () => {
|
||||||
|
const h = harness('soniox'); h.open();
|
||||||
|
h.message({ tokens: [{ text: 'Before expiry.', is_final: true }], finished: true });
|
||||||
|
assert.equal(h.events.at(-1).isFinal, true);
|
||||||
|
assert.equal(h.events.at(-1).text, 'Before expiry.');
|
||||||
|
assert.equal(h.timers.size, 1);
|
||||||
|
h.tick(); h.open();
|
||||||
|
assert.equal(h.sockets.length, 2);
|
||||||
|
h.client.stop();
|
||||||
|
});
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const vm = require('node:vm');
|
||||||
|
const ts = require('typescript');
|
||||||
|
|
||||||
|
// Execute production TypeScript with only the native boundary and clock stubbed.
|
||||||
|
exports.loader = (globals = {}, mocks = {}) => {
|
||||||
|
const cache = new Map();
|
||||||
|
function load(file) {
|
||||||
|
file = path.resolve(__dirname, '../..', file);
|
||||||
|
if (cache.has(file)) return cache.get(file);
|
||||||
|
const output = {};
|
||||||
|
cache.set(file, output);
|
||||||
|
const source = ts.transpileModule(fs.readFileSync(file, 'utf8'), {
|
||||||
|
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||||
|
}).outputText;
|
||||||
|
vm.runInNewContext(source, {
|
||||||
|
exports: output, console, Uint8Array, ...globals,
|
||||||
|
require(name) {
|
||||||
|
if (name in mocks) return mocks[name];
|
||||||
|
if (!name.startsWith('.')) throw new Error(`Unmocked module: ${name}`);
|
||||||
|
return load(path.resolve(path.dirname(file), name + '.ts'));
|
||||||
|
},
|
||||||
|
}, { filename: file });
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
return load;
|
||||||
|
};
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { PhoneGestureRecognizer } = require('../.test-build/app/phone-ui/phone-gestures.js');
|
||||||
|
function setup() {
|
||||||
|
let now = 0, id = 0;
|
||||||
|
const timers = new Map(), events = [];
|
||||||
|
const r = new PhoneGestureRecognizer((type, x, y) => events.push({ type, x, y }), {
|
||||||
|
set(fn, ms) { timers.set(++id, { fn, at: now + ms }); return id; },
|
||||||
|
clear(id) { timers.delete(id); },
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
r, events,
|
||||||
|
touch: (action, x = 20, y = 30, pointers = 1) => r.touch({ action, x, y, pointers }),
|
||||||
|
advance(ms) {
|
||||||
|
now += ms;
|
||||||
|
for (const [id, timer] of [...timers]) if (timer.at <= now && timers.delete(id)) timer.fn();
|
||||||
|
},
|
||||||
|
types: () => events.map(e => e.type),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
test('single tap retains mirror coordinates and waits to disambiguate double tap', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('up');
|
||||||
|
assert.deepEqual(s.types(), []); s.advance(280);
|
||||||
|
assert.deepEqual(s.events, [{ type: 'tap', x: 20, y: 30 }]);
|
||||||
|
});
|
||||||
|
test('double tap emits back without accidental selection', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('up'); s.advance(150);
|
||||||
|
s.touch('down'); s.touch('up'); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), ['double-tap']);
|
||||||
|
});
|
||||||
|
test('a hold emits a press and release, never a tap', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
|
||||||
|
});
|
||||||
|
test('tap then hold emits the app-menu gesture without an earlier selection', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('up'); s.advance(100);
|
||||||
|
s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), ['short-then-long-press', 'long-press-release']);
|
||||||
|
});
|
||||||
|
test('four directional swipes cancel pending holds and taps', () => {
|
||||||
|
for (const [x, y, expected] of [[80, 30, 'swipe-right'], [-40, 30, 'swipe-left'], [20, 90, 'swipe-down'], [20, -30, 'swipe-up']]) {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('move', x, y); s.advance(500); s.touch('up', x, y); s.advance(500);
|
||||||
|
assert.deepEqual(s.types(), [expected]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('two fingers send back exactly once and cancel the long-press timer', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('down', 21, 30, 2); s.advance(500);
|
||||||
|
s.touch('up', 21, 30, 2); s.touch('up'); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), ['double-tap']);
|
||||||
|
});
|
||||||
|
test('cancellation prevents delayed selections and releases active holds', () => {
|
||||||
|
const s = setup(); s.touch('down'); s.touch('up'); s.r.cancel(); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), []);
|
||||||
|
s.touch('down'); s.advance(500); s.touch('cancel'); s.advance(1000);
|
||||||
|
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
|
||||||
|
});
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
|
||||||
|
const rect = (x, y, width, height) => ({ x, y, width, height });
|
||||||
|
function surface(c, id, box, values, zOrder = 0, transparency = 'opaque') {
|
||||||
|
c.configureSurface(id, { ...box, zOrder, transparency });
|
||||||
|
c.submitSurfaceFrame(id, Uint8Array.from(values), rect(0, 0, box.width, box.height));
|
||||||
|
}
|
||||||
|
test('shell color key preserves underlying text while opaque black covers it', () => {
|
||||||
|
const c = new SurfaceCompositor(3, 2);
|
||||||
|
surface(c, 'app', rect(0, 0, 3, 2), [200, 200, 200, 200, 200, 200]);
|
||||||
|
surface(c, 'shell', rect(0, 0, 3, 2), [0, 1, 255, 0, 0, 0], 1, 'color-key');
|
||||||
|
c.setUnderlayDim(1, 0.5);
|
||||||
|
assert.deepEqual([...c.composite()], [100, 1, 255, 100, 100, 100]);
|
||||||
|
c.setSurfaceVisible('shell', false);
|
||||||
|
assert.deepEqual([...c.composite()], [100, 100, 100, 100, 100, 100]);
|
||||||
|
surface(c, 'opaque', rect(1, 0, 1, 2), [0, 0], 2);
|
||||||
|
assert.deepEqual([...c.composite()], [100, 0, 100, 100, 0, 100]);
|
||||||
|
});
|
||||||
|
test('partial frames clip in surface coordinates, then surfaces clip to the screen', () => {
|
||||||
|
const c = new SurfaceCompositor(3, 2);
|
||||||
|
surface(c, 'app', rect(-1, 0, 3, 2), [1, 2, 3, 4, 5, 6]);
|
||||||
|
const source = Uint8Array.from([99, 10, 20, 99, 30, 40, 99]);
|
||||||
|
c.submitSurfaceFrame('app', source.subarray(1, 5), rect(1, 1, 2, 2));
|
||||||
|
source.fill(0);
|
||||||
|
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
|
||||||
|
c.submitSurfaceFrame('app', Uint8Array.from([50, 60, 70, 80]), rect(-1, -1, 2, 2));
|
||||||
|
// The clipped update changed the offscreen column, leaving visible pixels intact.
|
||||||
|
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
|
||||||
|
});
|
||||||
|
test('moving a surface retains its frame; resizing clears it; blanking is reversible', () => {
|
||||||
|
const c = new SurfaceCompositor(3, 1);
|
||||||
|
surface(c, 'app', rect(0, 0, 2, 1), [7, 9]);
|
||||||
|
c.configureSurface('app', { ...rect(1, 0, 2, 1), zOrder: 0, transparency: 'opaque' });
|
||||||
|
assert.deepEqual([...c.composite()], [0, 7, 9]);
|
||||||
|
c.setScreenBlanked(true); assert.deepEqual([...c.composite()], [0, 0, 0]);
|
||||||
|
c.setScreenBlanked(false); assert.deepEqual([...c.composite()], [0, 7, 9]);
|
||||||
|
c.configureSurface('app', { ...rect(0, 0, 3, 1), zOrder: 0, transparency: 'opaque' });
|
||||||
|
assert.deepEqual([...c.composite()], [0, 0, 0]);
|
||||||
|
});
|
||||||
|
test('layer order is independent of creation order; removal reveals lower surfaces', () => {
|
||||||
|
const c = new SurfaceCompositor(1, 1);
|
||||||
|
surface(c, 'top', rect(0, 0, 1, 1), [20], 10);
|
||||||
|
surface(c, 'bottom', rect(0, 0, 1, 1), [80], -1);
|
||||||
|
assert.equal(c.composite()[0], 20);
|
||||||
|
c.removeSurface('top'); assert.equal(c.composite()[0], 80);
|
||||||
|
assert.throws(() => c.submitSurfaceFrame('missing', new Uint8Array(1), rect(0, 0, 1, 1)));
|
||||||
|
assert.throws(() => c.submitSurfaceFrame('bottom', new Uint8Array(2), rect(0, 0, 1, 1)));
|
||||||
|
});
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { loader } = require('./helpers/load-typescript.cjs');
|
||||||
|
const load = loader();
|
||||||
|
const { TranscriptModel, TranscriptScroll, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts');
|
||||||
|
const { SpeechPauseDetector } = load('app/native/speech-pause.ts');
|
||||||
|
const { TimedTranscript } = load('app/native/transcript-format.ts');
|
||||||
|
|
||||||
|
function pcm(amplitude, ms = 50) {
|
||||||
|
const bytes = new Uint8Array(ms * 16 * 2);
|
||||||
|
for (let i = 0; i < bytes.length; i += 2) {
|
||||||
|
bytes[i] = amplitude & 255;
|
||||||
|
bytes[i + 1] = (amplitude >> 8) & 255;
|
||||||
|
}
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('partials replace, finals correct, and pause breaks survive display and saving', () => {
|
||||||
|
const m = new TranscriptModel();
|
||||||
|
m.accept({ text: 'Hello wor', isFinal: false });
|
||||||
|
m.accept({ text: 'Hello world.', isFinal: true, paragraphBreakAfter: true });
|
||||||
|
m.accept({ text: 'Next', isFinal: false });
|
||||||
|
m.accept({ text: 'Next paragraph.', isFinal: false });
|
||||||
|
assert.equal(m.text, 'Hello world.\nNext paragraph.');
|
||||||
|
m.accept({ text: 'Next paragraph!', isFinal: true });
|
||||||
|
assert.equal(m.text, 'Hello world.\nNext paragraph!');
|
||||||
|
m.accept({ text: '', isFinal: true });
|
||||||
|
assert.equal(m.text, 'Hello world.\nNext paragraph!');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('onboard pauses can precede or follow finalization', () => {
|
||||||
|
for (const before of [true, false]) {
|
||||||
|
const m = new TranscriptModel();
|
||||||
|
m.accept({ text: 'first', isFinal: false });
|
||||||
|
if (before) m.pause();
|
||||||
|
m.accept({ text: 'First.', isFinal: true });
|
||||||
|
if (!before) m.pause();
|
||||||
|
m.accept({ text: 'Second.', isFinal: false });
|
||||||
|
assert.equal(m.text, 'First.\nSecond.');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('formatting metadata is used only by Transcribe, with revisable live speaker tokens', () => {
|
||||||
|
const final = new TimedTranscript();
|
||||||
|
final.append({ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1' });
|
||||||
|
const preview = final.copy();
|
||||||
|
preview.append({ text: ' Wrong', start_ms: 600, end_ms: 900, speaker: '2' });
|
||||||
|
assert.equal(preview.text, 'Hello.\nWrong');
|
||||||
|
assert.equal(final.text, 'Hello.');
|
||||||
|
final.append({ text: ' Right.', start_ms: 600, end_ms: 900, speaker: '1' });
|
||||||
|
final.append({ text: ' Later.', start_ms: 2400, end_ms: 2800, speaker: '1' });
|
||||||
|
assert.equal(final.text, 'Hello. Right.\nLater.');
|
||||||
|
const event = { text: 'Hello. Right. Later.', transcribeText: final.text, isFinal: false };
|
||||||
|
const m = new TranscriptModel();
|
||||||
|
m.accept(event);
|
||||||
|
assert.equal(m.text, final.text);
|
||||||
|
assert.equal(event.text.includes('\n'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wrapping preserves explicit newlines, blank paragraphs, and long unspaced words', () => {
|
||||||
|
assert.deepEqual([...wrapTranscribeText(s => s.length, 'one two\nthree\n\nabcdefghij', 5)],
|
||||||
|
['one', 'two', 'three', '', 'abcde', 'fghij']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('scroll follows the tail until scrolled back, then resumes only at the bottom', () => {
|
||||||
|
const s = new TranscriptScroll();
|
||||||
|
assert.equal(s.layout(30, 10), 20);
|
||||||
|
assert.equal(s.layout(33, 10), 23);
|
||||||
|
s.scroll(-3);
|
||||||
|
assert.equal(s.layout(50, 10), 20);
|
||||||
|
assert.equal(s.layout(51, 10), 20);
|
||||||
|
s.scroll(100);
|
||||||
|
assert.equal(s.layout(55, 10), 45);
|
||||||
|
s.scroll(-100);
|
||||||
|
assert.equal(s.layout(60, 10), 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('short transcripts become scrollable without disabling automatic following', () => {
|
||||||
|
const s = new TranscriptScroll();
|
||||||
|
assert.equal(s.layout(3, 10), 0);
|
||||||
|
s.scroll(-3);
|
||||||
|
assert.equal(s.layout(12, 10), 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores leading silence', () => {
|
||||||
|
const d = new SpeechPauseDetector();
|
||||||
|
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
|
||||||
|
d.accept(pcm(3000));
|
||||||
|
for (let i = 0; i < 29; i++) assert.equal(d.accept(pcm(0)), false);
|
||||||
|
assert.equal(d.accept(pcm(0)), true);
|
||||||
|
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
|
||||||
|
d.accept(pcm(3000));
|
||||||
|
assert.equal(d.accept(pcm(0, 1500)), true);
|
||||||
|
});
|
||||||
@@ -11,6 +11,8 @@
|
|||||||
"outDir": "../.test-build"
|
"outDir": "../.test-build"
|
||||||
},
|
},
|
||||||
"include": [
|
"include": [
|
||||||
|
"../app/graphics/surface-compositor.ts",
|
||||||
|
"../app/phone-ui/phone-gestures.ts",
|
||||||
"../app/g2/glasses-hardware-identity.ts",
|
"../app/g2/glasses-hardware-identity.ts",
|
||||||
"../app/g2/ble-proximity.ts",
|
"../app/g2/ble-proximity.ts",
|
||||||
"../app/g2/even-advertisement.ts",
|
"../app/g2/even-advertisement.ts",
|
||||||
|
|||||||
Reference in new issue
Block a user