forked from Blendman974/faceclaw
Add stats collection and first-run on-glasses onboarding app
This commit is contained in:
1 parent
84baee9c76
commit
406cefe0c5
73 files changed
+3906
-98
No files matched your search
@@ -0,0 +1,69 @@
|
||||
import type { Pool } from "pg";
|
||||
import type { Report } from "./report.ts";
|
||||
|
||||
/**
|
||||
* Stores one report and its rows in a single transaction. The phone keeps
|
||||
* resending a report until it sees a 2xx, so a report id that is already
|
||||
* stored is a retry whose response was lost, not new data.
|
||||
*/
|
||||
export async function saveReport(pool: Pool, report: Report): Promise<"saved" | "duplicate"> {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const inserted = await client.query(
|
||||
`insert into reports
|
||||
(report_id, install_id, created_at, schema_version, level, app_version, platform, build, payload)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
on conflict (report_id) do nothing`,
|
||||
[
|
||||
report.reportId,
|
||||
report.installId,
|
||||
report.createdAt,
|
||||
report.schema,
|
||||
report.level,
|
||||
report.app.version,
|
||||
report.app.platform,
|
||||
report.app.build,
|
||||
JSON.stringify(report),
|
||||
],
|
||||
);
|
||||
if (inserted.rowCount === 0) {
|
||||
await client.query("rollback");
|
||||
return "duplicate";
|
||||
}
|
||||
|
||||
const days: string[] = [];
|
||||
const names: string[] = [];
|
||||
const values: number[] = [];
|
||||
for (const [day, counts] of Object.entries(report.counters)) {
|
||||
for (const [name, value] of Object.entries(counts)) {
|
||||
days.push(day);
|
||||
names.push(name);
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
if (names.length) {
|
||||
await client.query(
|
||||
`insert into report_counters (report_id, day, name, value)
|
||||
select $1, day::date, name, value from unnest($2::text[], $3::text[], $4::bigint[]) as t(day, name, value)`,
|
||||
[report.reportId, days, names, values],
|
||||
);
|
||||
}
|
||||
|
||||
const keys = Object.keys(report.snapshot);
|
||||
if (keys.length) {
|
||||
await client.query(
|
||||
`insert into report_snapshot (report_id, key, value)
|
||||
select $1, key, value from unnest($2::text[], $3::jsonb[]) as t(key, value)`,
|
||||
[report.reportId, keys, keys.map((key) => JSON.stringify(report.snapshot[key]))],
|
||||
);
|
||||
}
|
||||
await client.query("commit");
|
||||
return "saved";
|
||||
} catch (error) {
|
||||
await client.query("rollback").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* The analytics report format the phone app uploads (PROTOCOL.md), and its
|
||||
* validation. The app-side producer is app/analytics/analytics-report.ts;
|
||||
* keep the two in step.
|
||||
*
|
||||
* Everything is checked against tight limits so a malformed or hostile
|
||||
* client can't store much: unknown fields are ignored, and anything
|
||||
* out of range rejects the whole report.
|
||||
*/
|
||||
|
||||
export const REPORT_SCHEMA_VERSION = 1;
|
||||
|
||||
export type ReportLevel = "minimal" | "full";
|
||||
export type SnapshotValue = boolean | number | string;
|
||||
|
||||
export interface Report {
|
||||
schema: number;
|
||||
reportId: string;
|
||||
installId: string;
|
||||
level: ReportLevel;
|
||||
/** When the phone sealed the report (ISO 8601). */
|
||||
createdAt: string;
|
||||
app: { version: string; platform: string; build: string };
|
||||
/** UTC day (YYYY-MM-DD) → counter name → count. */
|
||||
counters: Record<string, Record<string, number>>;
|
||||
/** Point-in-time state (devices, API-key presence, settings), flat namespaced keys. */
|
||||
snapshot: Record<string, SnapshotValue>;
|
||||
}
|
||||
|
||||
export class ReportError extends Error {}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const VERSION = /^[0-9A-Za-z.+-]{1,32}$/;
|
||||
const TOKEN = /^[a-z0-9-]{1,24}$/;
|
||||
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
||||
const COUNTER_NAME = /^[a-z0-9][a-z0-9_.:-]{0,119}$/;
|
||||
// Snapshot keys embed the app's settings keys, which are camelCase.
|
||||
const SNAPSHOT_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,119}$/;
|
||||
|
||||
const MAX_DAYS = 100;
|
||||
const MAX_COUNTERS = 2000;
|
||||
const MAX_COUNT = 1_000_000_000;
|
||||
const MAX_SNAPSHOT_KEYS = 500;
|
||||
const MAX_SNAPSHOT_STRING = 200;
|
||||
|
||||
export function parseReport(value: unknown): Report {
|
||||
const body = record(value, "report");
|
||||
if (body.schema !== REPORT_SCHEMA_VERSION) throw new ReportError(`unsupported schema ${String(body.schema)}`);
|
||||
const level = body.level;
|
||||
if (level !== "minimal" && level !== "full") throw new ReportError("bad level");
|
||||
const app = record(body.app, "app");
|
||||
return {
|
||||
schema: REPORT_SCHEMA_VERSION,
|
||||
reportId: uuid(body.reportId, "reportId"),
|
||||
installId: uuid(body.installId, "installId"),
|
||||
level,
|
||||
createdAt: timestamp(body.createdAt, "createdAt"),
|
||||
app: {
|
||||
version: matching(app.version, VERSION, "app.version"),
|
||||
platform: matching(app.platform, TOKEN, "app.platform"),
|
||||
build: matching(app.build, TOKEN, "app.build"),
|
||||
},
|
||||
counters: counters(body.counters ?? {}),
|
||||
snapshot: snapshot(body.snapshot ?? {}),
|
||||
};
|
||||
}
|
||||
|
||||
function counters(value: unknown): Report["counters"] {
|
||||
const days = record(value, "counters");
|
||||
const dayKeys = Object.keys(days);
|
||||
if (dayKeys.length > MAX_DAYS) throw new ReportError("too many days");
|
||||
const result: Report["counters"] = {};
|
||||
let total = 0;
|
||||
for (const day of dayKeys) {
|
||||
if (!DAY.test(day) || Number.isNaN(Date.parse(`${day}T00:00:00Z`))) throw new ReportError(`bad day ${day}`);
|
||||
const entries = record(days[day], `counters.${day}`);
|
||||
const counts: Record<string, number> = {};
|
||||
for (const [name, count] of Object.entries(entries)) {
|
||||
if (!COUNTER_NAME.test(name)) throw new ReportError(`bad counter name ${name}`);
|
||||
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0 || count > MAX_COUNT) {
|
||||
throw new ReportError(`bad count for ${name}`);
|
||||
}
|
||||
if (++total > MAX_COUNTERS) throw new ReportError("too many counters");
|
||||
counts[name] = count;
|
||||
}
|
||||
result[day] = counts;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function snapshot(value: unknown): Report["snapshot"] {
|
||||
const entries = Object.entries(record(value, "snapshot"));
|
||||
if (entries.length > MAX_SNAPSHOT_KEYS) throw new ReportError("too many snapshot keys");
|
||||
const result: Report["snapshot"] = {};
|
||||
for (const [key, item] of entries) {
|
||||
if (!SNAPSHOT_KEY.test(key)) throw new ReportError(`bad snapshot key ${key}`);
|
||||
const ok =
|
||||
typeof item === "boolean" ||
|
||||
(typeof item === "number" && Number.isFinite(item)) ||
|
||||
(typeof item === "string" && item.length <= MAX_SNAPSHOT_STRING);
|
||||
if (!ok) throw new ReportError(`bad snapshot value for ${key}`);
|
||||
result[key] = item;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function record(value: unknown, what: string): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new ReportError(`${what} must be an object`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function uuid(value: unknown, what: string): string {
|
||||
const text = typeof value === "string" ? value.toLowerCase() : "";
|
||||
if (!UUID.test(text)) throw new ReportError(`bad ${what}`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function matching(value: unknown, pattern: RegExp, what: string): string {
|
||||
if (typeof value !== "string" || !pattern.test(value)) throw new ReportError(`bad ${what}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, what: string): string {
|
||||
if (typeof value !== "string" || value.length > 40 || Number.isNaN(Date.parse(value))) throw new ReportError(`bad ${what}`);
|
||||
return new Date(value).toISOString();
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { saveReport } from "./db.ts";
|
||||
import { parseReport, ReportError } from "./report.ts";
|
||||
|
||||
/**
|
||||
* Receives the phone app's daily analytics reports (PROTOCOL.md) and stores
|
||||
* them in Postgres. Runs behind nginx, which terminates TLS and rate-limits;
|
||||
* the database connection comes from the standard PG* environment variables.
|
||||
*
|
||||
* Nothing about the request itself is stored or logged: no IP address, no
|
||||
* headers, only the report body.
|
||||
*/
|
||||
|
||||
const PORT = Number(process.env.PORT ?? 8290);
|
||||
const HOST = process.env.HOST ?? "127.0.0.1";
|
||||
const MAX_BODY_BYTES = 64 * 1024;
|
||||
|
||||
const pool = new Pool({ max: 4 });
|
||||
pool.on("error", (error) => console.error("idle database client failed:", error.message));
|
||||
|
||||
const server = createServer((request, response) => {
|
||||
handle(request, response).catch((error: unknown) => {
|
||||
console.error("request failed:", error instanceof Error ? error.message : error);
|
||||
if (!response.headersSent) reply(response, 500, { error: "internal error" });
|
||||
});
|
||||
});
|
||||
|
||||
async function handle(request: IncomingMessage, response: ServerResponse): Promise<void> {
|
||||
const path = (request.url ?? "/").split("?")[0];
|
||||
if (path === "/v1/report") {
|
||||
if (request.method !== "POST") return reply(response, 405, { error: "POST only" });
|
||||
if (Number(request.headers["content-length"] ?? 0) > MAX_BODY_BYTES) {
|
||||
response.setHeader("connection", "close");
|
||||
return reply(response, 413, { error: "too large" });
|
||||
}
|
||||
const body = await readBody(request);
|
||||
if (body === null) return reply(response, 413, { error: "too large" });
|
||||
let report;
|
||||
try {
|
||||
report = parseReport(JSON.parse(body));
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError || error instanceof ReportError) {
|
||||
return reply(response, 400, { error: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await saveReport(pool, report);
|
||||
// Room for more later (e.g. the latest released version, for update checks).
|
||||
return reply(response, 200, { ok: true });
|
||||
}
|
||||
if (path === "/healthz" && request.method === "GET") {
|
||||
await pool.query("select 1");
|
||||
return reply(response, 200, { ok: true });
|
||||
}
|
||||
reply(response, 404, { error: "not found" });
|
||||
}
|
||||
|
||||
/**
|
||||
* The request body as text, or null if it is over MAX_BODY_BYTES. An
|
||||
* oversized body is still read to the end (and dropped) so the client gets
|
||||
* the 413 instead of a reset connection; nginx caps it before it gets here.
|
||||
*/
|
||||
function readBody(request: IncomingMessage): Promise<string | null> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
request.on("data", (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size <= MAX_BODY_BYTES) chunks.push(chunk);
|
||||
});
|
||||
request.on("end", () => resolve(size > MAX_BODY_BYTES ? null : Buffer.concat(chunks).toString("utf8")));
|
||||
request.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function reply(response: ServerResponse, status: number, body: object): void {
|
||||
response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" });
|
||||
response.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
function shutdown(): void {
|
||||
server.close(() => {
|
||||
void pool.end().finally(() => process.exit(0));
|
||||
});
|
||||
setTimeout(() => process.exit(0), 5000).unref();
|
||||
}
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
server.listen(PORT, HOST, () => console.log(`faceclaw-stats listening on ${HOST}:${PORT}`));
|
||||
Reference in new issue
Block a user