From f0bb727a8024b5f3de02a94fd9a92f619e42b18b Mon Sep 17 00:00:00 2001 From: Guillaume ETHEVE Date: Sat, 7 Nov 2026 23:15:29 +0100 Subject: [PATCH 1/3] Add opt-in authenticated HTTPS export of R1 health history --- .../app/FaceclawHealthExportSecret.kt | 47 +++++++++ app/bootstrap.android.ts | 2 + app/health/health-export.ts | 72 ++++++++++++++ app/health/health-store.ts | 37 +++++++ app/native/health-export-secret.ts | 13 +++ app/ui/dashboard-settings.ts | 36 ++++++- app/ui/dashboard/settings-menus.ts | 14 +++ docs/validation-health-export.md | 31 ++++++ tests/assistant-settings-visibility.test.cjs | 1 + tests/health-export.test.cjs | 96 +++++++++++++++++++ 10 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHealthExportSecret.kt create mode 100644 app/health/health-export.ts create mode 100644 app/native/health-export-secret.ts create mode 100644 docs/validation-health-export.md create mode 100644 tests/health-export.test.cjs diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHealthExportSecret.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHealthExportSecret.kt new file mode 100644 index 00000000..a5052518 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHealthExportSecret.kt @@ -0,0 +1,47 @@ +package com.faceclaw.app + +import android.content.Context +import android.security.keystore.KeyGenParameterSpec +import android.security.keystore.KeyProperties +import android.util.AtomicFile +import java.io.File +import java.security.KeyStore +import javax.crypto.Cipher +import javax.crypto.KeyGenerator +import javax.crypto.SecretKey +import javax.crypto.spec.GCMParameterSpec + +/** This credential cannot be restored onto a device lacking its Keystore key. */ +object FaceclawHealthExportSecret { + private const val ALIAS = "faceclaw.health.export" + private fun file(context: Context) = AtomicFile(File(context.noBackupFilesDir, "health-export-key")) + private fun key(): SecretKey { + val store = KeyStore.getInstance("AndroidKeyStore").apply { load(null) } + (store.getKey(ALIAS, null) as? SecretKey)?.let { return it } + return KeyGenerator.getInstance(KeyProperties.KEY_ALGORITHM_AES, "AndroidKeyStore").apply { + init(KeyGenParameterSpec.Builder(ALIAS, KeyProperties.PURPOSE_ENCRYPT or KeyProperties.PURPOSE_DECRYPT) + .setBlockModes(KeyProperties.BLOCK_MODE_GCM) + .setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE).build()) + }.generateKey() + } + @JvmStatic @Synchronized fun read(context: Context): String { + val held = file(context) + if (!held.baseFile.exists()) return "" + return try { + val bytes = held.readFully() + val cipher = Cipher.getInstance("AES/GCM/NoPadding") + cipher.init(Cipher.DECRYPT_MODE, key(), GCMParameterSpec(128, bytes.copyOfRange(0, 12))) + String(cipher.doFinal(bytes.copyOfRange(12, bytes.size)), Charsets.UTF_8) + } catch (_: Exception) { "" } // Require re-entry after key loss; never log the credential. + } + @JvmStatic @Synchronized fun write(context: Context, value: String) { + val held = file(context) + if (value.isEmpty()) { held.delete(); return } + val cipher = Cipher.getInstance("AES/GCM/NoPadding") + cipher.init(Cipher.ENCRYPT_MODE, key()) + val bytes = cipher.iv + cipher.doFinal(value.toByteArray(Charsets.UTF_8)) + val stream = held.startWrite() + try { stream.write(bytes); held.finishWrite(stream) } + catch (error: Exception) { held.failWrite(stream); throw error } + } +} diff --git a/app/bootstrap.android.ts b/app/bootstrap.android.ts index 82c23dd5..d08a5588 100644 --- a/app/bootstrap.android.ts +++ b/app/bootstrap.android.ts @@ -1,3 +1,4 @@ +import { startHealthExport } from "./health/health-export" import { Application } from '@nativescript/core' import { registerShareIntentHandler } from './native/share-intents' import { installNativeUserAgent } from './util/http' @@ -18,6 +19,7 @@ Application.on(Application.launchEvent, () => { // they are closed would be lost if the process restarted first, so this keeps // them reaching disk regardless. No-op when there is nothing new. startLiveHealthSync() +startHealthExport() // Collect from the ring on a wall-clock cadence (:01 and :31). Without this // nothing drives a pull on a timer at all - only onRingReady() on reconnect - diff --git a/app/health/health-export.ts b/app/health/health-export.ts new file mode 100644 index 00000000..2bf0a54b --- /dev/null +++ b/app/health/health-export.ts @@ -0,0 +1,72 @@ +import { healthStore } from "./health-store-files"; +import { isFixtureData } from "./health-seed"; +import { healthExportEnabledSetting, healthExportUrlSetting, healthExportKeySetting } from "../ui/dashboard-settings"; +import { getStringSetting, setStringSetting } from "../native/settings-store"; +import { fetchTextWithUserAgent } from "../util/http"; + +let running = false; +let started = false; +let status = "Not synchronized"; +export function healthExportStatus(): string { return status; } + +/** One bounded batch, with durable acknowledgement and no overlapping requests. */ +export async function syncHealthExport(): Promise { + if (running) return; + running = true; + try { + if (!healthExportEnabledSetting.get()) { status = "Export disabled"; return; } + if (isFixtureData()) { status = "Sample data is never exported"; return; } + const url = healthExportUrlSetting.get(); + const key = healthExportKeySetting.get(); + if (!url || healthExportUrlSetting.validationError() || !key) { + status = "Configure an HTTPS endpoint and API key"; + return; + } + const generation = getStringSetting("health.export.generation", "0"); + let held: { url?: string; generation?: string; offsets?: Record } = {}; + try { held = JSON.parse(getStringSetting("health.export.cursor", "{}")) ?? {}; } catch { /* First sync. */ } + const offsets = held.url === url && held.generation === generation ? held.offsets ?? {} : {}; + let limit = 200; + let batch = healthStore().exportBatch(offsets, limit); + let body = JSON.stringify({ version: 1, samples: batch.samples, sleep: batch.sleep }); + // Normalized records contain only ASCII keys, numbers and metric names. + // Large sleep histories must fit the server body limit as well as its record limit. + while (body.length > 240_000 && limit > 1) { + limit = Math.max(1, Math.floor(limit / 2)); + batch = healthStore().exportBatch(offsets, limit); + body = JSON.stringify({ version: 1, samples: batch.samples, sleep: batch.sleep }); + } + if (body.length > 240_000) { status = "A health record exceeds the export size limit"; return; } + if (!batch.scanned) { status = "Up to date"; return; } + if (batch.samples.length || batch.sleep.length) { + status = "Sending health data"; + const response = await fetchTextWithUserAgent(url, { + method: "POST", headers: { "Authorization": `Bearer ${key}`, "Content-Type": "application/json" }, + body, + timeoutMs: 30_000, dontFollowRedirects: true, + }); + if (response.status !== 200) { + status = response.status === 401 || response.status === 403 + ? "API key refused; check configuration" : `Server refused batch (HTTP ${response.status}); will retry`; + return; + } + const ack = await response.json<{ accepted?: boolean }>(); + if (ack.accepted !== true) { status = "Invalid server acknowledgement; will retry"; return; } + } + // An edit during the request must not acknowledge data on a different destination. + if (url !== healthExportUrlSetting.get() || generation !== getStringSetting("health.export.generation", "0")) return; + setStringSetting("health.export.cursor", JSON.stringify({ url, generation, offsets: batch.cursor })); + status = "Batch synchronized; remaining history follows automatically"; + } catch { + // Errors may include request headers or data: never log or display them. + status = "Synchronization failed; will retry automatically"; + } finally { running = false; } +} + +/** Android app process only; no extra BLE pulls, wake locks, or background service. */ +export function startHealthExport(): void { + if (started) return; + started = true; + setTimeout(() => { void syncHealthExport(); }, 10_000); + setInterval(() => { void syncHealthExport(); }, 60_000); +} diff --git a/app/health/health-store.ts b/app/health/health-store.ts index beb93e7a..6c07d041 100644 --- a/app/health/health-store.ts +++ b/app/health/health-store.ts @@ -228,6 +228,43 @@ export class HealthStore { // ------------------------------------------------------------------------- // Reading + /** Append-journal cursor, committed by the exporter only after server acknowledgement. + * Revisions appended to an old month are exported too; a time watermark would lose them. + */ + exportBatch(cursor: Record, limit = 200): { + cursor: Record; samples: HealthSample[]; sleep: SleepSession[]; scanned: number; + } { + const next = { ...cursor }; + const samples: HealthSample[] = []; + const sleep: SleepSession[] = []; + let scanned = 0; + for (const name of this.backend.list().filter(candidate => /^samples-\d{4}-\d{2}\.jsonl$/.test(candidate) || candidate === SLEEP_FILE).sort()) { + const text = this.backend.read(name); + if (text === null) throw new Error("Health export could not read history"); + let offset = next[name] ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0 || offset > text.length) offset = 0; + while (scanned < limit) { + const end = text.indexOf("\n", offset); + if (end < 0) break; // Never consume a partially written final record. + const line = text.slice(offset, end); + offset = end + 1; + scanned++; + if (name === SLEEP_FILE) { + try { + const session = JSON.parse(line) as SleepSession; + if (session?.timeResolved === true) sleep.push(session); + } catch { /* Ignore incomplete/corrupt journal lines as the local reader does. */ } + } else { + const sample = fromLine(line); + if (sample) samples.push(sample); + } + } + next[name] = offset; + if (scanned >= limit) break; + } + return { cursor: next, samples, sleep, scanned }; + } + /** Every stored sample overlapping [startMs, endMs). Exact - reads shards. */ samplesInRange(startMs: number, endMs: number): HealthSample[] { const out: HealthSample[] = []; diff --git a/app/native/health-export-secret.ts b/app/native/health-export-secret.ts new file mode 100644 index 00000000..baf22dec --- /dev/null +++ b/app/native/health-export-secret.ts @@ -0,0 +1,13 @@ +import { Utils } from "@nativescript/core"; +declare const com: any; + +/** Android Keystore-backed, excluded from backups. Never falls back to plain settings. */ +export function readHealthExportSecret(): string { + if (!global.isAndroid) return ""; + return String(com.faceclaw.app.FaceclawHealthExportSecret.read(Utils.android.getApplicationContext())); +} + +export function writeHealthExportSecret(value: string): void { + if (!global.isAndroid) throw new Error("Health export is available on Android only"); + com.faceclaw.app.FaceclawHealthExportSecret.write(Utils.android.getApplicationContext(), value); +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 6bed4672..18e2877d 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -1,3 +1,4 @@ +import { readHealthExportSecret, writeHealthExportSecret } from "../native/health-export-secret"; import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts"; import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures"; import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve"; @@ -260,6 +261,39 @@ export class ConfigSettingString extends ConfigSett } +/** A changed credential starts a new delivery history without retaining the old secret. */ +class HealthExportKeySetting extends ConfigSettingString { + get(): string { return readHealthExportSecret(); } + set(value: string): string { + const normalized = value.trim(); + if (normalized !== this.get()) { + writeHealthExportSecret(normalized); + setStringSetting("health.export.cursor", ""); + setStringSetting("health.export.generation", String(Number(getStringSetting("health.export.generation", "0")) + 1)); + } + return normalized; + } + getValidValue(): string { return this.get(); } +} + +export const healthExportEnabledSetting = new ConfigSettingBoolean({ + id: "healthExportEnabled", label: "Export health data", storageKey: "health.export.enabled", + defaultValue: false, reportInAnalytics: false, + description: "Send saved R1 health history to your server over HTTPS. Retries while Faceclaw is running; phone storage is retained.", +}); +export const healthExportUrlSetting = new ConfigSettingString({ + id: "healthExportUrl", label: "Endpoint URL", storageKey: "health.export.url", defaultValue: "", + normalize: value => (value ?? "").trim(), + validate: value => !value || /^https:\/\/[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?(?::\d{1,5})?(?:\/[^\s?#]*)?$/i.test(value) + ? null : "Use an HTTPS URL without credentials, query or fragment", + description: "Full ingestion URL, for example https://health.undefined-dev.fr/v1/health", +}); +export const healthExportKeySetting = new HealthExportKeySetting({ + id: "healthExportKey", label: "API key", storageKey: "health.export.key", defaultValue: "", + inputKind: "password", formatValue: value => value ? "Configured" : "(empty)", + description: "Device-specific write-only key, stored using Android Keystore.", +}); + export const batteryDisplayModeSetting = new ConfigSettingEnum({ id: "batteryDisplayMode", label: "Style", @@ -1680,7 +1714,7 @@ export class EditTextSettingLayer implements Layer { for (let index = 0; index < message.length; index++) { image.drawText(font, 22, messageTop + index * step, message[index]!, 200); } - image.drawText(font, 22, messageTop + (message.length + 2) * step, truncateSetting(this.setting.get(), 52), 220); + image.drawText(font, 22, messageTop + (message.length + 2) * step, truncateSetting(this.setting.inputKind === "password" ? this.setting.displayValue() : this.setting.get(), 52), 220); const error = this.setting.validationError(); if (error) wrapText(font, error, width - 48).forEach((line, index) => image.drawText(font, 22, messageTop + (message.length + 4 + index) * step, line, 180)); diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index f974bb8f..e8b844f4 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -1,3 +1,4 @@ +import { healthExportStatus, syncHealthExport } from "../../health/health-export"; import { remoteInputMenuItem } from "./remote-input-menu"; import { knownFolders } from "@nativescript/core"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; @@ -28,6 +29,9 @@ import type { LayerContext } from "../layers"; import { drawRightValueMenuItem, openModalMenu, submenuItem, type MenuItem } from "../menu"; import { shell } from "../shell/shell"; import { + healthExportEnabledSetting, + healthExportUrlSetting, + healthExportKeySetting, anthropicApiKeySetting, appSwitcherPositionSetting, statusBarPositionSetting, @@ -187,6 +191,16 @@ function settingsSections(): SettingsSection[] { ]; }, }, + ...(!global.isIOS ? [{ + label: "Health export", + get items() { return [ + toggleSettingMenuItem(healthExportEnabledSetting), + textSettingMenuItem(healthExportUrlSetting), + textSettingMenuItem(healthExportKeySetting), + { label: "Sync now", description: healthExportStatus(), onSelect: () => { void syncHealthExport(); } }, + { label: healthExportStatus(), disabled: true, onSelect: () => {} }, + ]; }, + }] : []), { label: "API Keys", items: [ diff --git a/docs/validation-health-export.md b/docs/validation-health-export.md new file mode 100644 index 00000000..e7fda93a --- /dev/null +++ b/docs/validation-health-export.md @@ -0,0 +1,31 @@ +# R1 health export + +Android Settings → Health export uses the existing toggle and text/password +editors. Export defaults off; URL and API key default empty. Only the API key +uses AES-GCM with Android Keystore and an AtomicFile in noBackupFilesDir. Keys +are masked on both glasses and phone. The URL/cursor use shared settings. + +The matching API/manifests live in Infrastructure/FaceclawHealth and SQL dashboards +in GrafanaDashboards/Health. Protocol v1 sends normalized HealthSample/SleepSession +fields with original millisecond timestamps. Batches contain at most 200 records +and 240 KB. HTTPS is mandatory, redirects are refused, and only HTTP 200 with +accepted:true advances the cursor. No health content or keys are logged. + +The exporter reads existing append journals, including revisions to old months, +without extra BLE pulls. Partial trailing lines are retried. Unresolved sleep +sessions and sample fixtures are excluded. Replayed data is upserted server-side. +A new URL/key resets delivery history safely. One batch runs per minute while +the Android process is alive; this adds no background service or wake lock and +does not promise synchronization while the app is stopped. Disable preserves +local history and existing server data. Watch/iOS export is outside this change. + +Software validation: exporter tests for replay, revisions, failed/malformed +acknowledgements, redirects, destination/credential edits, fixtures, partial +records and overlapping requests; full JS suite, TypeScript and Android release +compilation. API tests additionally cover PostgreSQL rollback and deduplication. + +Hardware/end-to-end validation remains pending: enable ring health sync, configure +the deployed URL/key, then compare measured timestamps/values and sleep durations +with Grafana. Test offline/reconnect, app restart, disable/re-enable and rotated +keys. Check key masking, log redaction and preservation of other preferences. +No firmware modification is needed. diff --git a/tests/assistant-settings-visibility.test.cjs b/tests/assistant-settings-visibility.test.cjs index 43e96713..a349a113 100644 --- a/tests/assistant-settings-visibility.test.cjs +++ b/tests/assistant-settings-visibility.test.cjs @@ -40,6 +40,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') { }, }); const load = loader({ global: { isIOS: false, isAndroid: true } }, { + '../../health/health-export': { healthExportStatus: () => 'Export disabled', syncHealthExport() {} }, './remote-input-menu': { remoteInputMenuItem: () => ({ label: 'Remote input' }) }, '@nativescript/core': { knownFolders: {} }, '../../graphics/ui-fonts': { getDefaultSmallFont: () => ({}) }, diff --git a/tests/health-export.test.cjs b/tests/health-export.test.cjs new file mode 100644 index 00000000..ed948469 --- /dev/null +++ b/tests/health-export.test.cjs @@ -0,0 +1,96 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +function setup() { + const files = new Map(); + const backend = { list: () => [...files.keys()], read: name => files.get(name) ?? null, + exists: name => files.has(name), append: (name, text) => files.set(name, (files.get(name) ?? '') + text), + write: (name, text) => files.set(name, text) }; + const { HealthStore } = loader()('app/health/health-store.ts'); + const store = new HealthStore(backend); + const settings = new Map(); + let url = 'https://health.example/v1/health', key = 'private-test-key', enabled = true, fixture = false; + let response = { status: 200, json: async () => ({ accepted: true }) }; + const requests = []; + const exporter = loader({ setTimeout, setInterval }, { + './health-store-files': { healthStore: () => store }, + './health-seed': { isFixtureData: () => fixture }, + '../ui/dashboard-settings': { + healthExportEnabledSetting: { get: () => enabled }, + healthExportUrlSetting: { get: () => url, validationError: () => url.startsWith('https://') ? null : 'HTTPS required' }, + healthExportKeySetting: { get: () => key }, + }, + '../native/settings-store': { getStringSetting: (k, d) => settings.get(k) ?? d, setStringSetting: (k, v) => settings.set(k, v) }, + '../util/http': { fetchTextWithUserAgent: async (url, options) => { requests.push({ url, options }); return typeof response === 'function' ? response() : response; } }, + })('app/health/health-export.ts'); + return { store, files, settings, exporter, requests, + configure: options => { if ('url' in options) url=options.url; if ('enabled' in options) enabled=options.enabled; if ('fixture' in options) fixture=options.fixture; if ('response' in options) response=options.response; } }; +} +const sample = { metric:'heartRate',startMs:1700000000000,spanMs:3600000,min:60,max:90,avg:75,total:0 }; + +test('exports original measurements, acknowledges durably and catches revisions to old buckets', async () => { + const h=setup();h.store.ingestSamples([sample]); + await h.exporter.syncHealthExport(); + assert.equal(h.requests.length,1); + assert.deepEqual(JSON.parse(h.requests[0].options.body),{version:1,samples:[sample],sleep:[]}); + assert.equal(h.requests[0].options.dontFollowRedirects,true); + assert.ok(!h.settings.get('health.export.cursor').includes('private-test-key')); + await h.exporter.syncHealthExport();assert.equal(h.requests.length,1); + h.store.ingestSamples([{...sample,avg:80}]);await h.exporter.syncHealthExport(); + assert.equal(JSON.parse(h.requests[1].options.body).samples[0].avg,80); + h.configure({url:'https://another.example/v1/health'});await h.exporter.syncHealthExport(); + assert.equal(h.requests.length,3); +}); + +test('failed or malformed acknowledgements retain cursor for replay; no secret in status', async () => { + for (const response of [{status:401},{status:503},{status:302},{status:200,json:async()=>({})},()=>Promise.reject(new Error('private-test-key'))]) { + const h=setup();h.store.ingestSamples([sample]);h.configure({response});await h.exporter.syncHealthExport(); + assert.equal(h.settings.has('health.export.cursor'),false); + assert.ok(!h.exporter.healthExportStatus().includes('private-test-key')); + h.configure({response:{status:200,json:async()=>({accepted:true})}}); + await h.exporter.syncHealthExport();assert.equal(h.requests.length,2); + } +}); + +test('disabled export, fixtures, missing key configuration and HTTP never upload', async () => { + for (const config of [{enabled:false},{fixture:true},{url:'http://health.example/v1/health'},{url:''}]) { + const h=setup();h.store.ingestSamples([sample]);h.configure(config); + await h.exporter.syncHealthExport();assert.equal(h.requests.length,0); + } +}); + +test('bounded batches preserve partial lines, unresolved sleep and late appends', () => { + const h=setup();h.store.ingestSamples([sample]); + const name=[...h.files.keys()].find(n=>n.startsWith('samples-')); + const first=h.store.exportBatch({},1);assert.equal(first.scanned,1); + const second=h.store.exportBatch(first.cursor,200);assert.equal(second.samples.length,1); + h.files.set(name,h.files.get(name)+'{"m":'); + const partial=h.store.exportBatch(second.cursor);assert.equal(partial.scanned,0); + h.files.set(name,h.files.get(name)+'"hrv","t":1700000000000,"s":3600000,"n":1,"x":3,"a":2,"u":0}\n'); + assert.equal(h.store.exportBatch(partial.cursor).samples[0].metric,'hrv'); + h.files.set('sleep.jsonl',JSON.stringify({timeResolved:false,startMs:12})+'\n'); + assert.equal(h.store.exportBatch({}).sleep.length,0); +}); + +test('concurrent ticks and changed credentials cannot incorrectly acknowledge a batch', async () => { + const h=setup();h.store.ingestSamples([sample]);let release; + h.configure({response:()=>new Promise(resolve=>{release=resolve;})}); + const first=h.exporter.syncHealthExport();await h.exporter.syncHealthExport(); + assert.equal(h.requests.length,1); + h.settings.set('health.export.generation','1');release({status:200,json:async()=>({accepted:true})}); + await first;assert.equal(h.settings.has('health.export.cursor'),false); +}); + +test('large sleep histories split into bounded bodies without losing sessions', async () => { + const h=setup(); + const sessions=Array.from({length:6},(_,i)=>({timeResolved:true,startMs:1700000000000+i*86400000, + endMs:1700028800000+i*86400000,dayStartMs:1699920000000+i*86400000, + totalSec:28000,wakeSec:800,remSec:4000,lightSec:18000,deepSec:6000, + segments:Array.from({length:2000},()=>({stageId:2,halfMinutes:1}))})); + h.files.set('sleep.jsonl',sessions.map(s=>JSON.stringify(s)+'\n').join('')); + for(let i=0;i<10;i++) await h.exporter.syncHealthExport(); + assert.ok(h.requests.length>1); + assert.ok(h.requests.every(r=>r.options.body.length<=240000)); + assert.equal(h.requests.reduce((n,r)=>n+JSON.parse(r.options.body).sleep.length,0),sessions.length); +}); -- 2.54.1 From 8d57b7c1d043c3e26dcaa6c4511511af41b47185 Mon Sep 17 00:00:00 2001 From: Guillaume ETHEVE Date: Sat, 7 Nov 2026 23:43:32 +0100 Subject: [PATCH 2/3] Document the separate health API repository --- docs/validation-health-export.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/validation-health-export.md b/docs/validation-health-export.md index e7fda93a..2e298cdf 100644 --- a/docs/validation-health-export.md +++ b/docs/validation-health-export.md @@ -5,8 +5,9 @@ editors. Export defaults off; URL and API key default empty. Only the API key uses AES-GCM with Android Keystore and an AtomicFile in noBackupFilesDir. Keys are masked on both glasses and phone. The URL/cursor use shared settings. -The matching API/manifests live in Infrastructure/FaceclawHealth and SQL dashboards -in GrafanaDashboards/Health. Protocol v1 sends normalized HealthSample/SleepSession +The API/image pipeline lives in the dedicated Infrastructure/FaceclawHealth Git +repository. Kubernetes manifests live in the Infrastructure repository under +FaceclawHealth, and SQL dashboards in GrafanaDashboards/Health. Protocol v1 sends normalized HealthSample/SleepSession fields with original millisecond timestamps. Batches contain at most 200 records and 240 KB. HTTPS is mandatory, redirects are refused, and only HTTP 200 with accepted:true advances the cursor. No health content or keys are logged. -- 2.54.1 From a575db8e27583b2daa7838003cdc8bb92c18eb7d Mon Sep 17 00:00:00 2001 From: Guillaume ETHEVE Date: Sun, 8 Nov 2026 01:13:55 +0100 Subject: [PATCH 3/3] feat: retain and export latest heart-rate measurements --- app/health/health-ingest.ts | 19 +++++++ app/health/health-live.ts | 31 ++++++----- app/health/health-store.ts | 14 +++-- app/health/health-types.ts | 6 +++ docs/validation-health-export.md | 39 ++++++++++++++ tests/health-export.test.cjs | 26 +++++++++ tests/health-heart-rate-measurements.test.cjs | 48 +++++++++++++++++ tests/health-live.test.cjs | 53 +++++++++++++++++++ 8 files changed, 218 insertions(+), 18 deletions(-) create mode 100644 tests/health-heart-rate-measurements.test.cjs diff --git a/app/health/health-ingest.ts b/app/health/health-ingest.ts index 70afcfc0..aca95ce9 100644 --- a/app/health/health-ingest.ts +++ b/app/health/health-ingest.ts @@ -48,6 +48,7 @@ import { HOUR_MS, + HEART_RATE_MEASUREMENT_SPAN_MS, TEN_MINUTES_MS, type HealthSample, type SleepSession, @@ -65,6 +66,8 @@ export type WireHourlyRecord = { * zero bytes instead. */ anchorMs: number | null; + /** Latest reading, only from live pages; raw journal replay does not guess its clock. */ + latest?: { timestampSeconds: number; value: number; receivedAtMs: number; clockOffsetMs: number }; groups: readonly { hourIndex: number; avg: number; max: number; min: number }[]; }; @@ -146,6 +149,22 @@ export function convertRecords(records: readonly WireRecord[]): ConversionResult * guesses. Dropping loses a page; guessing corrupts the history silently. */ function convertHourly(record: WireHourlyRecord, result: ConversionResult): void { + if (record.metric === "heartRate" && record.latest) { + const { timestampSeconds, value, receivedAtMs, clockOffsetMs } = record.latest; + const startMs = timestampSeconds * 1000 - clockOffsetMs; + // Invert the existing handshake clock shift, never replace measurement time + // with reception time or snap it onto an invented ten-minute bucket. + // Older pages can mix clock domains: only accept a recent live reading. + if (Number.isSafeInteger(timestampSeconds) && Number.isSafeInteger(receivedAtMs) + && Number.isFinite(clockOffsetMs) && Number.isInteger(value) && value > 0 && value <= 300 + && startMs >= 946684800000 && startMs <= receivedAtMs + 120_000 + && receivedAtMs - startMs <= 20 * 60_000) { + result.samples.push({ metric: "heartRate", startMs, spanMs: HEART_RATE_MEASUREMENT_SPAN_MS, + min: value, max: value, avg: value, total: value }); + } else { + result.skipped.push({ what: "latest heart rate", why: "invalid, stale or unresolved measurement timestamp/value" }); + } + } if (record.anchorMs === null) { result.skipped.push({ what: `${record.metric} page, ${record.groups.length} groups`, diff --git a/app/health/health-live.ts b/app/health/health-live.ts index 7dc86c04..0c1b74d0 100644 --- a/app/health/health-live.ts +++ b/app/health/health-live.ts @@ -240,7 +240,7 @@ function accumulateStepBuckets( return merged; } -function toWire(record: any): WireRecord | null { +function toWire(record: any, includeLatest: boolean): WireRecord | null { const cmdHi = Number(record.cmdHi); const metric = HOURLY_METRIC[cmdHi]; @@ -256,7 +256,12 @@ function toWire(record: any): WireRecord | null { min: Number(group.min), }); } - return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups }; + const receivedAtMs = Number(record.receivedAtMs); + const latest = includeLatest && metric === "heartRate" && Number(record.tagRaw) > 0 ? { + timestampSeconds: Number(record.tagRaw), value: Number(record.current), receivedAtMs, + clockOffsetMs: ringClockOffsetMs(receivedAtMs), + } : undefined; + return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups, latest }; } if (cmdHi === CMD_STEPS) { @@ -391,12 +396,12 @@ type StoreResult = Omit; * fails, so the caller keeps whatever it would otherwise mark as consumed. * Shared by the live batch and the journal replay. */ -function storeRecords(records: JavaRecordList): StoreResult { +function storeRecords(records: JavaRecordList, includeLatest = true): StoreResult { const wire: WireRecord[] = []; const size = records.size(); for (let i = 0; i < size; i++) { try { - const converted = toWire(records.get(i)); + const converted = toWire(records.get(i), includeLatest); if (converted) wire.push(converted); } catch (error) { console.warn("health live: skipped an undecodable record", error); @@ -480,7 +485,7 @@ function replayFrameJournalOnce(): void { const records: JavaRecordList = ringProtocol.decodeFrameJournal(File.fromPath(journalPath).readTextSync(), after); const size = records.size(); if (size > 0) { - const result = storeRecords(records); + const result = storeRecords(records, false); let through = after; for (let i = 0; i < size; i++) through = Math.max(through, Number(records.get(i).receivedAtMs)); const next: ReplayCursor = { version: JOURNAL_DECODER_VERSION, throughReceivedAtMs: through }; @@ -543,7 +548,7 @@ export function syncLiveRecords(): LiveSyncResult { /** * Ask the ring for a fresh pull now, rather than waiting out the automatic - * 30-minute cycle. Returns immediately; the pull itself takes ~15s on the + * ten-minute cycle. Returns immediately; the pull itself takes ~15s on the * communicator's worker thread and lands through the next `syncLiveRecords()`. * * Throttled communicator-side to one a minute — see @@ -584,19 +589,17 @@ export function startLiveHealthSync(): void { } /** - * Pull the ring on a wall-clock-aligned cadence: :01 and :31, every hour. + * Pull the ring every ten minutes, one minute after each bucket boundary. * - * Chris, 2026-09-12: *"I think we should be polling the ring on the 30 minute - * cycle (that was my intent, not a floor honestly)."* The 30-minute value in - * `RING_HEALTH_MIN_PULL_INTERVAL_MS` was always meant as a collection rhythm - * and got built as a throttle, so a connected, stable ring pulled never — - * nothing in the system drives a pull on a timer at all. + * This collects the latest heart-rate observation at roughly the cadence + * the stock app documents, as well as the existing historical buckets. + * It does not force the sensor to take a new measurement. * * Two reasons this is aligned to the wall clock rather than a plain interval: * * 1. **The ring's own step buckets close on 10-minute boundaries.** Pulling on * the boundary collects a just-closed bucket instead of a half-formed one. - * 2. **An elapsed timer drifts.** A pull at :07 sets the next at :37, then :07, + * 2. **An elapsed timer drifts.** A pull at :07 sets the next at :17, then :27, * wandering away from the boundary it is supposed to track. Re-arming * against the clock each time cannot drift. * @@ -606,7 +609,7 @@ export function startLiveHealthSync(): void { * This only *requests* a pull. The communicator still applies its own anti-spam * floor, so an extra call here can never hammer the ring. */ -const ALIGNED_PULL_MINUTES = [1, 31]; +const ALIGNED_PULL_MINUTES = [1, 11, 21, 31, 41, 51]; let alignedPullTimer: ReturnType | null = null; function msUntilNextAlignedPull(now: Date): number { diff --git a/app/health/health-store.ts b/app/health/health-store.ts index 6c07d041..5eeaab20 100644 --- a/app/health/health-store.ts +++ b/app/health/health-store.ts @@ -59,6 +59,7 @@ import { type HealthSample, + isHeartRateMeasurement, type Rollup, type SampleMetric, type SleepSession, @@ -265,12 +266,14 @@ export class HealthStore { return { cursor: next, samples, sleep, scanned }; } - /** Every stored sample overlapping [startMs, endMs). Exact - reads shards. */ - samplesInRange(startMs: number, endMs: number): HealthSample[] { + /** Bucket samples in [startMs, endMs). Point observations are opt-in so + * existing charts and daily averages never combine two overlapping series. */ + samplesInRange(startMs: number, endMs: number, includeMeasurements = false): HealthSample[] { const out: HealthSample[] = []; for (const name of this.shardNamesInRange(startMs, endMs)) { for (const sample of this.loadShard(name).values()) { - if (sample.startMs >= startMs && sample.startMs < endMs) out.push(sample); + if (sample.startMs >= startMs && sample.startMs < endMs + && (includeMeasurements || !isHeartRateMeasurement(sample))) out.push(sample); } } out.sort((a, b) => a.startMs - b.startMs); @@ -334,6 +337,7 @@ export class HealthStore { for (const name of this.backend.list()) { if (!name.startsWith(SAMPLE_PREFIX) || !name.endsWith(SAMPLE_SUFFIX)) continue; for (const sample of this.loadShard(name).values()) { + if (isHeartRateMeasurement(sample)) continue; const key = dayKey(sample.startMs); let byMetric = perDay.get(key); if (!byMetric) { @@ -360,7 +364,9 @@ export class HealthStore { private updateRollupsFor(samples: readonly HealthSample[]): void { const file = this.loadRollups(); const touched = new Set(); - for (const sample of samples) touched.add(dayKey(sample.startMs)); + for (const sample of samples) { + if (!isHeartRateMeasurement(sample)) touched.add(dayKey(sample.startMs)); + } for (const key of touched) { const parts = key.split("-").map(Number); const dayStart = new Date(parts[0]!, parts[1]! - 1, parts[2]!).getTime(); diff --git a/app/health/health-types.ts b/app/health/health-types.ts index f9aeb698..e8713cc8 100644 --- a/app/health/health-types.ts +++ b/app/health/health-types.ts @@ -36,6 +36,12 @@ export function isCumulative(metric: SampleMetric): boolean { export const HOUR_MS = 3_600_000; export const TEN_MINUTES_MS = 600_000; +/** Point observation, not a ten-minute average. Distinct from hourly buckets. */ +export const HEART_RATE_MEASUREMENT_SPAN_MS = 1; + +export function isHeartRateMeasurement(sample: HealthSample): boolean { + return sample.metric === "heartRate" && sample.spanMs === HEART_RATE_MEASUREMENT_SPAN_MS; +} export const DAY_MS = 86_400_000; /** diff --git a/docs/validation-health-export.md b/docs/validation-health-export.md index 2e298cdf..e22c16c4 100644 --- a/docs/validation-health-export.md +++ b/docs/validation-health-export.md @@ -30,3 +30,42 @@ the deployed URL/key, then compare measured timestamps/values and sleep duration with Grafana. Test offline/reconnect, app restart, disable/re-enable and rotated keys. Check key masking, log redaction and preservation of other preferences. No firmware modification is needed. + +## Latest heart-rate observations + +Live HR pages also carry a last-measured timestamp and value. These are now +stored as `heartRate` with `spanMs: 1`, at the exact measurement timestamp: +this denotes a point observation, **not** a ten-minute average. Existing hourly +records retain `spanMs: 3600000`. The existing local and PostgreSQL keys include +span, so readings and summaries coexist without collisions. Equal BPM at a new +timestamp is a new measurement; repeated timestamps are deduplicated across +restarts and corrected values update the same observation. Local bucket charts +and daily rollups exclude points, including when the cache is rebuilt. + +The existing aligned ring pull changes from :01/:31 to :01/:11/:21/:31/:41/:51. +This is a collection cadence, not a sensor setting or a guarantee of six readings +per hour. It uses the same ring worker, connection windows and anti-spam guards; +it makes more collection attempts than before. Offline gaps remain gaps and +there is no new background service. The exporter still sends at most one batch +per minute while the Android process runs. + +Timestamp handling deliberately retains the current handshake's legacy clock +convention, inverting its existing phone-timezone offset once. Only a live point +within 20 minutes of page reception and no more than two minutes in the future +is accepted; invalid/missing timestamps and BPM outside 1–300 are skipped. +Reception time is never substituted for measurement time. Old frame-journal +replay continues restoring hourly history but does not reconstruct point readings: +old pages may mix clock domains. No firmware/time-sync migration is attempted. +A timezone change around measurement or an older value can therefore leave a gap; +correct placement across such transitions still requires hardware validation. + +The deployed v1 API already accepts span 1 and upserts on device/metric/time/span. +No API image, schema migration, Infrastructure change or key rotation is needed. +The Grafana overview shows separate hourly-average and latest-reading series, +with point markers for readings, not interpolated ten-minute averages. + +Software checks cover latest-page conversion, invalid/stale/future readings, +timezone offsets, restart/deduplication, unchanged hourly rollups, export and +all six scheduling slots. Hardware checks remain pending: record raw/current +measurement times across several pulls, compare BPM/time with the stock app, +and check reconnect, timezone/DST transitions and collection battery impact. diff --git a/tests/health-export.test.cjs b/tests/health-export.test.cjs index ed948469..44416356 100644 --- a/tests/health-export.test.cjs +++ b/tests/health-export.test.cjs @@ -94,3 +94,29 @@ test('large sleep histories split into bounded bodies without losing sessions', assert.ok(h.requests.every(r=>r.options.body.length<=240000)); assert.equal(h.requests.reduce((n,r)=>n+JSON.parse(r.options.body).sleep.length,0),sessions.length); }); + +test('point heart rates and hourly summaries export separately with original timestamps', async () => { + const h = setup(); + const point = { ...sample, spanMs: 1, startMs: sample.startMs + 123456, min: 72, max: 72, avg: 72, total: 72 }; + h.store.ingestSamples([sample, point]); + await h.exporter.syncHealthExport(); + const body = JSON.parse(h.requests[0].options.body); + assert.deepEqual(body.samples, [sample, point]); + h.store.resetCache(); + assert.equal(h.store.ingestSamples([point]), 0); + await h.exporter.syncHealthExport(); + assert.equal(h.requests.length, 1); +}); + +test('a corrected point replaces its old value without changing hourly summaries', async () => { + const h = setup(); + const point = { ...sample, spanMs: 1, min: 72, max: 72, avg: 72, total: 72 }; + h.store.ingestSamples([sample, point]); + await h.exporter.syncHealthExport(); + const correction = { ...point, min: 74, max: 74, avg: 74, total: 74 }; + h.store.ingestSamples([correction]); + await h.exporter.syncHealthExport(); + assert.deepEqual(JSON.parse(h.requests[1].options.body).samples, [correction]); + assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1, true).length, 2); + assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1)[0].avg, sample.avg); +}); diff --git a/tests/health-heart-rate-measurements.test.cjs b/tests/health-heart-rate-measurements.test.cjs new file mode 100644 index 00000000..bcc1456d --- /dev/null +++ b/tests/health-heart-rate-measurements.test.cjs @@ -0,0 +1,48 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); +const { convertRecords } = loader()('app/health/health-ingest.ts'); +const receivedAtMs = Date.UTC(2026, 10, 8, 12); +const page = { + kind: 'hourly', metric: 'heartRate', anchorMs: receivedAtMs, + groups: [{ hourIndex: 0, min: 50, max: 90, avg: 65 }], + latest: { timestampSeconds: (receivedAtMs - 60000) / 1000, value: 72, receivedAtMs, clockOffsetMs: 0 }, +}; + +test('latest observation remains a point, independent of hourly averages and page anchor', () => { + const result = convertRecords([{ ...page, anchorMs: null }]); + assert.equal(result.samples.length, 1); + assert.equal(result.samples[0].spanMs, 1); + assert.equal(result.samples[0].startMs, receivedAtMs - 60000); + assert.equal(result.samples[0].min, 72); + assert.equal(result.samples[0].max, 72); + assert.equal(result.samples[0].avg, 72); +}); + +test('legacy handshake timezone offset is inverted once, preserving exact timestamps', () => { + for (const clockOffsetMs of [-3600000, -7200000, 7 * 3600000, 0, -19800000]) { + const latest = { ...page.latest, clockOffsetMs, + timestampSeconds: (receivedAtMs - 60000 + clockOffsetMs) / 1000 }; + const samples = convertRecords([{ ...page, latest }]).samples; + assert.equal(samples[0].startMs, receivedAtMs - 60000); + assert.equal(samples[1].spanMs, 3600000); + } +}); + +test('invalid, stale and future latest values never corrupt hourly history', () => { + for (const patch of [ + { value: 0 }, { value: -1 }, { value: 301 }, { value: NaN }, { value: 72.5 }, + { timestampSeconds: -1 }, { timestampSeconds: 0 }, { timestampSeconds: NaN }, + { timestampSeconds: receivedAtMs / 1000 + 121 }, + { timestampSeconds: receivedAtMs / 1000 - 1201 }, + { receivedAtMs: NaN }, { clockOffsetMs: NaN }, + ]) { + const result = convertRecords([{ ...page, latest: { ...page.latest, ...patch } }]); + assert.equal(result.samples.length, 1, JSON.stringify(patch)); + assert.equal(result.samples[0].spanMs, 3600000); + assert.equal(result.skipped.length, 1); + } + for (const metric of ['spo2', 'hrv']) { + assert.equal(convertRecords([{ ...page, metric }]).samples.length, 1); + } +}); diff --git a/tests/health-live.test.cjs b/tests/health-live.test.cjs index 9b2b2e25..bbac1b6e 100644 --- a/tests/health-live.test.cjs +++ b/tests/health-live.test.cjs @@ -210,3 +210,56 @@ test("a failed journal replay is retried on the next sync and never blocks the l live.syncLiveRecords(); assert.equal(failures, -1, "replay was not retried"); }); + +test("live latest heart rate retains measurement time and survives restart without duplicates", () => { + const measuredAt = Math.floor(Date.now() / 1000) * 1000 - 60_000; + function sync(previous, time) { + const env = liveEnvironment(); + if (previous) for (const [name, text] of previous.files) env.files.set(name, text); + Object.assign(env.record, { + receivedAtMs: time + 60_000, + tagRaw: (time + new Date(time + 60_000).getTimezoneOffset() * 60_000) / 1000, + current: 72, + }); + env.load("health-live").syncLiveRecords(); + return env; + } + const first = sync(null, measuredAt); + const repeated = sync(first, measuredAt); + const next = sync(repeated, measuredAt + 600_000); + const store = next.load("health-store-files").healthStore(); + const batch = store.exportBatch({}); + const points = Array.from(batch.samples).filter(s => s.spanMs === 1); + assert.deepEqual(points.map(s => [s.startMs, s.avg]), [[measuredAt, 72], [measuredAt + 600_000, 72]]); + assert.equal(batch.samples.filter(s => s.spanMs === 3600000).length, 1); + assert.equal(store.samplesInRange(measuredAt - 86400000, measuredAt + 86400000).length, 1, + "existing health charts only see bucket samples"); + const counts = () => Array.from(store.dailyRollups('heartRate', measuredAt - 86400000, measuredAt + 86400000).values()).reduce((n, r) => n + r.count, 0); + assert.equal(counts(), 1); + store.rebuildRollups(); + assert.equal(counts(), 1, "rebuilding must not mix point readings with hourly aggregates"); +}); + +test("aligned health collection requests all six ten-minute slots without timer drift", () => { + const { msUntilNextAlignedPull, ALIGNED_PULL_MINUTES } = liveEnvironment().load('health-live').__alignedPullInternals; + assert.deepEqual(Array.from(ALIGNED_PULL_MINUTES), [1, 11, 21, 31, 41, 51]); + for (const minute of ALIGNED_PULL_MINUTES) { + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute, 0)), 600000); + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute - 1, 30)), 30000); + } + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, 59, 0)), 120000); +}); + +test("journal replay restores hourly history without assigning a clock to old point readings", () => { + const receivedAtMs = Math.floor(Date.now() / 1000) * 1000; + let replay; + const ringProtocol = { decodeFrameJournal: () => ({ size: () => 1, get: () => replay }) }; + const env = liveEnvironment({ ringProtocol }); + replay = { ...env.record, current: 80, receivedAtMs, + tagRaw: (receivedAtMs + new Date(receivedAtMs).getTimezoneOffset() * 60000) / 1000 }; + env.files.set('/health/ring-frames.jsonl', 'synthetic journal'); + env.load('health-live').syncLiveRecords(); + const samples = env.load('health-store-files').healthStore().exportBatch({}).samples; + assert.equal(samples.length, 1); + assert.equal(samples[0].spanMs, 3600000); +}); -- 2.54.1