forked from Blendman974/faceclaw
454 lines
16 KiB
TypeScript
454 lines
16 KiB
TypeScript
/**
|
|
* Durable local storage for decoded ring health records.
|
|
*
|
|
* ## Why this exists at all, and why it is not a cache
|
|
*
|
|
* Live testing on 2026-09-10 found that the ring appears to advance its own
|
|
* delivery watermark when it sends the RSP to a health request, whether or not
|
|
* the DATA page that should follow ever arrives - so a request that loses the
|
|
* race can PERMANENTLY DISCARD backlog, silently, with no error anywhere in
|
|
* the protocol (implement-return, section 11). The practical consequence:
|
|
* the moment a record is successfully decoded may be the only moment it is
|
|
* ever available. It has to hit durable storage right then.
|
|
*
|
|
* The existing `ringHealthRecords` list on `FaceclawBleCommunicator` is
|
|
* explicitly a capped, ephemeral ring buffer (256 records, oldest dropped) and
|
|
* is not a store. This is the store.
|
|
*
|
|
* ## The layout, and why it is shaped this way
|
|
*
|
|
* health/samples-YYYY-MM.jsonl append-only, one JSON object per line
|
|
* health/sleep.jsonl append-only, one session per line
|
|
* health/rollups.json DERIVED daily rollups - a rebuildable cache
|
|
*
|
|
* The raw shards are the truth and are never rewritten in place, only appended
|
|
* to. That is the property the paragraph above demands: a crash, a bad parse,
|
|
* or a bug in the rollup code can never destroy a record that was once
|
|
* received, because nothing ever rewrites the file that holds it.
|
|
*
|
|
* `rollups.json` is the answer to "pre-aggregate on write, or aggregate on
|
|
* render?" - and the answer is BOTH, at different layers, because they are
|
|
* asked by different screens:
|
|
*
|
|
* - The glasses glance and the by-hour charts want one day at fine
|
|
* resolution. That comes from raw shards, aggregated on render. At ~360
|
|
* samples a day the work is trivial, and it stays exact.
|
|
* - The by-day charts want weeks or months. Aggregating those on render
|
|
* means parsing every shard in the range - about 9 MB a year of JSONL, and
|
|
* growing without bound. So daily rollups are maintained on write and
|
|
* served straight from this one small file, and a quarter-long chart never
|
|
* opens a shard at all.
|
|
*
|
|
* Because the rollup file is derived, it carries no risk: if it is missing,
|
|
* corrupt, or written by an older version, `rebuildRollups()` regenerates it
|
|
* from the shards. Deleting it is always safe.
|
|
*
|
|
* ## No new dependency
|
|
*
|
|
* SQLite is not a dependency of this project and adding one for this would be
|
|
* the largest change in the feature. Everything the phone view needs is a
|
|
* key-ordered scan over a few tens of thousands of rows, which is what the
|
|
* above does with the `File` API already used by `open-apps-persistence.ts`
|
|
* for the same kind of job.
|
|
*
|
|
* This module has NO NativeScript imports - the filesystem arrives through
|
|
* `HealthStorageBackend` - so its durability and dedupe semantics run under
|
|
* plain node in `tests/health-store.test.cjs`. The real backend is
|
|
* `health-store-files.ts`.
|
|
*/
|
|
|
|
import {
|
|
type HealthSample,
|
|
type Rollup,
|
|
type SampleMetric,
|
|
type SleepSession,
|
|
SAMPLE_METRICS,
|
|
monthKey,
|
|
rollupOf,
|
|
startOfLocalDay,
|
|
} from "./health-types";
|
|
|
|
export interface HealthStorageBackend {
|
|
exists(name: string): boolean;
|
|
read(name: string): string | null;
|
|
/** Must create the file if absent. Durable before returning. */
|
|
append(name: string, text: string): void;
|
|
write(name: string, text: string): void;
|
|
/** Every file in the health folder. Used to rebuild the rollup cache. */
|
|
list(): string[];
|
|
}
|
|
|
|
const SAMPLE_PREFIX = "samples-";
|
|
const SAMPLE_SUFFIX = ".jsonl";
|
|
const SLEEP_FILE = "sleep.jsonl";
|
|
const ROLLUP_FILE = "rollups.json";
|
|
const ROLLUP_VERSION = 1;
|
|
|
|
/** On-disk sample line. Short keys: this is the file that grows. */
|
|
type SampleLine = {
|
|
m: SampleMetric;
|
|
t: number;
|
|
s: number;
|
|
n: number;
|
|
x: number;
|
|
a: number;
|
|
u: number;
|
|
};
|
|
|
|
type RollupFile = {
|
|
version: number;
|
|
/** `YYYY-MM-DD` -> metric -> rollup. */
|
|
days: Record<string, Partial<Record<SampleMetric, Rollup>>>;
|
|
};
|
|
|
|
function sampleKey(sample: { metric: SampleMetric; startMs: number; spanMs: number }): string {
|
|
return `${sample.metric}|${sample.startMs}|${sample.spanMs}`;
|
|
}
|
|
|
|
function dayKey(ms: number): string {
|
|
const date = new Date(startOfLocalDay(ms));
|
|
const month = `${date.getMonth() + 1}`.padStart(2, "0");
|
|
const day = `${date.getDate()}`.padStart(2, "0");
|
|
return `${date.getFullYear()}-${month}-${day}`;
|
|
}
|
|
|
|
function shardName(ms: number): string {
|
|
return `${SAMPLE_PREFIX}${monthKey(ms)}${SAMPLE_SUFFIX}`;
|
|
}
|
|
|
|
export class HealthStore {
|
|
/** Shard name -> dedupe key -> sample. Loaded lazily, per shard. */
|
|
private readonly shards = new Map<string, Map<string, HealthSample>>();
|
|
private sleep: SleepSession[] | null = null;
|
|
private rollups: RollupFile | null = null;
|
|
private rollupsDirty = false;
|
|
|
|
constructor(private readonly backend: HealthStorageBackend) {}
|
|
|
|
/** Forget cached data after the backing files have been purged. */
|
|
resetCache(): void {
|
|
this.shards.clear();
|
|
this.sleep = null;
|
|
this.rollups = null;
|
|
this.rollupsDirty = false;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Writing
|
|
|
|
/**
|
|
* Store decoded samples. Durable before this returns.
|
|
*
|
|
* Returns how many were genuinely new or changed. Re-storing a bucket that
|
|
* is already held is a no-op on disk, which matters because the live poller
|
|
* re-requests the current hour every 30 minutes and would otherwise append a
|
|
* duplicate line each time.
|
|
*
|
|
* ## This no-op is real, and it is the ONLY place it needs to be
|
|
*
|
|
* Re-checked 2026-09-12 while chasing a shard that reached 22k lines in a
|
|
* day. The suspicion was that identical re-writes were being appended; they
|
|
* were not. `sameSample` below compares every stored field, `spanMs` is part
|
|
* of the key, and an unchanged sample never reaches `append`.
|
|
*
|
|
* What was actually appending was a genuinely CHANGING value: the steps
|
|
* ledger merged overlapping records with last-write-wins, and those records
|
|
* disagreed about each bucket's calorie fields, so one day's calorie total
|
|
* ping-ponged 643 <-> 671 and wrote two new lines every sync cycle. An
|
|
* append-only store recording a value that really does flip-flop is correct
|
|
* behaviour; the fix belonged upstream, in `health-live.ts`' merge, and is
|
|
* there now (max-by-index, order-independent).
|
|
*
|
|
* So: do NOT add a second dedupe layer at a caller or in the file backend.
|
|
* If the shard grows again, the value is changing and the question is why.
|
|
*/
|
|
ingestSamples(samples: readonly HealthSample[]): number {
|
|
const byShard = new Map<string, Map<string, HealthSample>>();
|
|
for (const sample of samples) {
|
|
if (!Number.isFinite(sample.startMs) || sample.spanMs <= 0) continue;
|
|
const name = shardName(sample.startMs);
|
|
const shard = this.loadShard(name);
|
|
const key = sampleKey(sample);
|
|
const pending = byShard.get(name) ?? new Map<string, HealthSample>();
|
|
const existing = pending.get(key) ?? shard.get(key);
|
|
if (existing && sameSample(existing, sample)) continue;
|
|
pending.set(key, sample);
|
|
byShard.set(name, pending);
|
|
}
|
|
let written = 0;
|
|
for (const [name, pending] of byShard) {
|
|
const list = [...pending.values()];
|
|
const text = list.map((sample) => `${JSON.stringify(toLine(sample))}\n`).join("");
|
|
// A failed append can leave an incomplete final line. Start each batch
|
|
// on a fresh line so retrying cannot concatenate a valid record onto it.
|
|
this.backend.append(name, `\n${text}`);
|
|
this.rollupsDirty = true;
|
|
// Only durable values participate in deduplication. If another shard
|
|
// fails, a retry skips this successful append and retries the rest.
|
|
const shard = this.loadShard(name);
|
|
for (const [key, sample] of pending) shard.set(key, sample);
|
|
written += list.length;
|
|
}
|
|
if (this.rollupsDirty) {
|
|
this.updateRollupsFor(samples);
|
|
this.rollupsDirty = false;
|
|
}
|
|
return written;
|
|
}
|
|
|
|
/**
|
|
* Store sleep sessions. Durable before this returns.
|
|
*
|
|
* Deduped on the session's own start time, so a night re-delivered by a
|
|
* later sync replaces rather than duplicates.
|
|
*/
|
|
ingestSleep(sessions: readonly SleepSession[]): number {
|
|
const held = this.loadSleep().slice();
|
|
const fresh: SleepSession[] = [];
|
|
for (const session of sessions) {
|
|
const index = held.findIndex(
|
|
(candidate) =>
|
|
candidate.startMs === session.startMs && candidate.timeResolved === session.timeResolved,
|
|
);
|
|
if (index >= 0) {
|
|
if (JSON.stringify(held[index]) === JSON.stringify(session)) continue;
|
|
held[index] = session;
|
|
} else {
|
|
held.push(session);
|
|
}
|
|
fresh.push(session);
|
|
}
|
|
if (fresh.length > 0) {
|
|
this.backend.append(SLEEP_FILE, "\n" + fresh.map((s) => `${JSON.stringify(s)}\n`).join(""));
|
|
this.sleep = held;
|
|
}
|
|
return fresh.length;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Reading
|
|
|
|
/** Every stored sample overlapping [startMs, endMs). Exact - reads shards. */
|
|
samplesInRange(startMs: number, endMs: number): HealthSample[] {
|
|
const out: HealthSample[] = [];
|
|
for (const name of this.shardNamesInRange(startMs, endMs)) {
|
|
for (const sample of this.loadShard(name).values()) {
|
|
if (sample.startMs >= startMs && sample.startMs < endMs) out.push(sample);
|
|
}
|
|
}
|
|
out.sort((a, b) => a.startMs - b.startMs);
|
|
return out;
|
|
}
|
|
|
|
/** Sleep sessions, newest first. */
|
|
sleepSessions(): SleepSession[] {
|
|
return this.loadSleep().slice().sort((a, b) => b.startMs - a.startMs);
|
|
}
|
|
|
|
/**
|
|
* Daily rollups over a range, served from the derived cache.
|
|
*
|
|
* This is the path a month- or quarter-long chart takes, and it deliberately
|
|
* never opens a sample shard. A day with no entry comes back absent, and the
|
|
* caller renders a gap.
|
|
*/
|
|
dailyRollups(metric: SampleMetric, startMs: number, endMs: number): Map<number, Rollup> {
|
|
const file = this.loadRollups();
|
|
const out = new Map<number, Rollup>();
|
|
let cursor = startOfLocalDay(startMs);
|
|
let guard = 0;
|
|
while (cursor < endMs && guard < 2000) {
|
|
guard += 1;
|
|
const entry = file.days[dayKey(cursor)]?.[metric];
|
|
if (entry) out.set(cursor, entry);
|
|
const next = new Date(cursor);
|
|
next.setDate(next.getDate() + 1);
|
|
cursor = next.getTime();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Local midnight of the earliest day with any stored data, or null. */
|
|
earliestDayMs(): number | null {
|
|
const file = this.loadRollups();
|
|
const keys = Object.keys(file.days).sort();
|
|
if (keys.length === 0) return null;
|
|
const parts = keys[0]!.split("-").map(Number);
|
|
return new Date(parts[0]!, parts[1]! - 1, parts[2]!).getTime();
|
|
}
|
|
|
|
/** How many sample rows are held, across every loaded shard. */
|
|
loadedSampleCount(): number {
|
|
let total = 0;
|
|
for (const shard of this.shards.values()) total += shard.size;
|
|
return total;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// The derived cache
|
|
|
|
/**
|
|
* Rebuild `rollups.json` from the raw shards. Safe to call at any time; the
|
|
* recovery path when the cache is missing, corrupt or from an older version.
|
|
*/
|
|
rebuildRollups(): void {
|
|
const days: RollupFile["days"] = {};
|
|
const perDay = new Map<string, Map<SampleMetric, HealthSample[]>>();
|
|
for (const name of this.backend.list()) {
|
|
if (!name.startsWith(SAMPLE_PREFIX) || !name.endsWith(SAMPLE_SUFFIX)) continue;
|
|
for (const sample of this.loadShard(name).values()) {
|
|
const key = dayKey(sample.startMs);
|
|
let byMetric = perDay.get(key);
|
|
if (!byMetric) {
|
|
byMetric = new Map();
|
|
perDay.set(key, byMetric);
|
|
}
|
|
const list = byMetric.get(sample.metric);
|
|
if (list) list.push(sample);
|
|
else byMetric.set(sample.metric, [sample]);
|
|
}
|
|
}
|
|
for (const [key, byMetric] of perDay) {
|
|
const entry: Partial<Record<SampleMetric, Rollup>> = {};
|
|
for (const metric of SAMPLE_METRICS) {
|
|
const rolled = rollupOf(byMetric.get(metric) ?? []);
|
|
if (rolled) entry[metric] = rolled;
|
|
}
|
|
days[key] = entry;
|
|
}
|
|
this.rollups = { version: ROLLUP_VERSION, days };
|
|
this.persistRollups();
|
|
}
|
|
|
|
private updateRollupsFor(samples: readonly HealthSample[]): void {
|
|
const file = this.loadRollups();
|
|
const touched = new Set<string>();
|
|
for (const sample of samples) touched.add(dayKey(sample.startMs));
|
|
for (const key of touched) {
|
|
const parts = key.split("-").map(Number);
|
|
const dayStart = new Date(parts[0]!, parts[1]! - 1, parts[2]!).getTime();
|
|
const dayEnd = new Date(parts[0]!, parts[1]! - 1, parts[2]! + 1).getTime();
|
|
const inDay = this.samplesInRange(dayStart, dayEnd);
|
|
const entry: Partial<Record<SampleMetric, Rollup>> = {};
|
|
for (const metric of SAMPLE_METRICS) {
|
|
const rolled = rollupOf(inDay.filter((sample) => sample.metric === metric));
|
|
if (rolled) entry[metric] = rolled;
|
|
}
|
|
file.days[key] = entry;
|
|
}
|
|
this.persistRollups();
|
|
}
|
|
|
|
private persistRollups(): void {
|
|
if (!this.rollups) return;
|
|
this.backend.write(ROLLUP_FILE, JSON.stringify(this.rollups));
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Loading
|
|
|
|
private shardNamesInRange(startMs: number, endMs: number): string[] {
|
|
const names: string[] = [];
|
|
const cursor = new Date(startOfLocalDay(startMs));
|
|
cursor.setDate(1);
|
|
let guard = 0;
|
|
while (cursor.getTime() < endMs && guard < 240) {
|
|
guard += 1;
|
|
names.push(`${SAMPLE_PREFIX}${monthKey(cursor.getTime())}${SAMPLE_SUFFIX}`);
|
|
cursor.setMonth(cursor.getMonth() + 1);
|
|
}
|
|
return names;
|
|
}
|
|
|
|
private loadShard(name: string): Map<string, HealthSample> {
|
|
const held = this.shards.get(name);
|
|
if (held) return held;
|
|
const shard = new Map<string, HealthSample>();
|
|
this.shards.set(name, shard);
|
|
const text = this.backend.exists(name) ? this.backend.read(name) : null;
|
|
if (!text) return shard;
|
|
for (const line of text.split("\n")) {
|
|
if (line.length === 0) continue;
|
|
const sample = fromLine(line);
|
|
// Later lines win: an append is how a re-synced bucket is corrected.
|
|
if (sample) shard.set(sampleKey(sample), sample);
|
|
}
|
|
return shard;
|
|
}
|
|
|
|
private loadSleep(): SleepSession[] {
|
|
if (this.sleep) return this.sleep;
|
|
const sessions: SleepSession[] = [];
|
|
const text = this.backend.exists(SLEEP_FILE) ? this.backend.read(SLEEP_FILE) : null;
|
|
if (text) {
|
|
for (const line of text.split("\n")) {
|
|
if (line.length === 0) continue;
|
|
try {
|
|
const parsed = JSON.parse(line) as SleepSession;
|
|
if (typeof parsed?.startMs !== "number") continue;
|
|
const index = sessions.findIndex((s) => s.startMs === parsed.startMs);
|
|
if (index >= 0) sessions[index] = parsed;
|
|
else sessions.push(parsed);
|
|
} catch {
|
|
// One unreadable line must not cost the rest of the file.
|
|
}
|
|
}
|
|
}
|
|
this.sleep = sessions;
|
|
return sessions;
|
|
}
|
|
|
|
private loadRollups(): RollupFile {
|
|
if (this.rollups) return this.rollups;
|
|
const text = this.backend.exists(ROLLUP_FILE) ? this.backend.read(ROLLUP_FILE) : null;
|
|
if (text) {
|
|
try {
|
|
const parsed = JSON.parse(text) as RollupFile;
|
|
if (parsed?.version === ROLLUP_VERSION && parsed.days) {
|
|
this.rollups = parsed;
|
|
return parsed;
|
|
}
|
|
} catch {
|
|
// Falls through to a rebuild - the cache is always reconstructible.
|
|
}
|
|
}
|
|
this.rollups = { version: ROLLUP_VERSION, days: {} };
|
|
this.rebuildRollups();
|
|
return this.rollups;
|
|
}
|
|
}
|
|
|
|
function sameSample(a: HealthSample, b: HealthSample): boolean {
|
|
return a.min === b.min && a.max === b.max && a.avg === b.avg && a.total === b.total;
|
|
}
|
|
|
|
function toLine(sample: HealthSample): SampleLine {
|
|
return {
|
|
m: sample.metric,
|
|
t: sample.startMs,
|
|
s: sample.spanMs,
|
|
n: sample.min,
|
|
x: sample.max,
|
|
a: sample.avg,
|
|
u: sample.total,
|
|
};
|
|
}
|
|
|
|
function fromLine(line: string): HealthSample | null {
|
|
try {
|
|
const parsed = JSON.parse(line) as SampleLine;
|
|
if (typeof parsed?.t !== "number" || typeof parsed?.m !== "string") return null;
|
|
return {
|
|
metric: parsed.m,
|
|
startMs: parsed.t,
|
|
spanMs: parsed.s,
|
|
min: parsed.n,
|
|
max: parsed.x,
|
|
avg: parsed.a,
|
|
total: parsed.u,
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|