Add stats collection and first-run on-glasses onboarding app

This commit is contained in:
James Babcock committed 2026-10-10 17:33:32 -07:00
1 parent 84baee9c76
commit 406cefe0c5
73 files changed
+3906 -98

No files matched your search

+92
View File
@@ -0,0 +1,92 @@
# Analytics report protocol
The phone app (app/analytics/) sends opt-in usage statistics to
`https://stats.faceclaw.org/v1/report`, at most about once a day per install,
plus hourly retries after a failure. Nothing is sent, or kept on the phone,
while Settings > Privacy > Data collection is None. What each level collects
is described in app/analytics/analytics-store.ts and in the privacy policy
(../PRIVACY); this file is the wire format.
## Request
`POST /v1/report` with a JSON body of at most 64 KiB:
```json
{
"schema": 1,
"reportId": "0f8fad5b-d9cb-469f-a165-70867728950e",
"installId": "7c9e6679-7425-40de-944b-e07fc1f90ae7",
"level": "minimal",
"createdAt": "2026-10-10T12:00:00.000Z",
"app": { "version": "0.8.3", "platform": "android", "build": "official" },
"counters": { "2026-10-10": { "g2.connect-attempt": 3, "g2.session-ready": 3 } },
"snapshot": { "device.g2.paired": true, "device.r1.paired": false }
}
```
- `reportId`: a random UUID per report. The phone resends a report until it
gets a 2xx, so the server stores each id once and treats repeats as
success.
- `installId`: a random UUID made when analytics is first turned on, deleted
(with everything else) when it is turned off. It ties one install's reports
together and nothing else.
- `level`: `minimal` or `full`, the setting when the report was sent. A
minimal report carries only minimal data.
- `createdAt`: when the phone sealed the report (its counters may cover
earlier days, e.g. after a failed upload).
- `app.build`: `official` (signed with the release key), `self-built` (a
release build signed with another key), `debug` (a debuggable build), or
`unknown`.
- `counters`: UTC day, then counter name (`[a-z0-9][a-z0-9_.:-]*`), then a
non-negative integer count.
- `snapshot`: state when the report was sealed, as flat keys
(`[A-Za-z0-9][A-Za-z0-9_.:-]*`) to a boolean, number or string of up to 200
characters.
The server ignores unknown fields and rejects (400) a report with anything
malformed or over its limits (100 days, 2000 counters, 500 snapshot keys).
## Response
`200 {"ok": true}` once stored (or already stored). Errors are
`{"error": "..."}` with 400 (malformed, not retried), 405, 413 (too large),
429 (rate-limited by nginx, retried) or 500 (retried). Later versions may add
fields to the 200 response, such as the latest released version for update
checks.
## Current counters and snapshot keys
Minimal level:
| Name | Kind | Meaning |
| --- | --- | --- |
| `g2.connect-attempt`, `g2.session-ready` | counter | Connection attempts to the glasses, and those that got a working session |
| `g2.failure.<reason>` | counter | Transport failures that dropped or aborted a session, by reason (e.g. `ack-timeout`, `write-failed`, `glasses-not-found`, `left-arm-not-found`, `heartbeat-ack-timeout`, `cfw-recovery-retry-limit`) |
| `g2.session-end.<cause>` | counter | Working sessions ending: `failure`, `arm-drop`, `requested` |
| `g2.session-length.<bucket>` | counter | How long ended sessions lasted: `under-1m`, `1m-10m`, `10m-1h`, `over-1h` |
| `g2.arm-drop.left`, `g2.arm-drop.right` | counter | An arm's BLE link dropping mid-session |
| `g2.disconnect-requested` | counter | Disconnects the app asked for |
| `g2.unpaired`, `g2.incompatible-firmware` | counter | Connections halted for a missing bond or incompatible firmware |
| `g2.ack-timeout`, `g2.cfw-replay`, `g2.cfw-nack` | counter | Recovered message-level problems |
| `g2.firmware-abnormal-exit`, `g2.firmware-system-exit` | counter | Firmware closing the app's page outside a shutdown |
| `g2.auth-unacknowledged`, `g2.even-app-conflict` | counter | Security auth not acked; write failures blamed on the Even app |
| `ring.direct.connect`, `ring.direct.disconnect` | counter | Direct (not via glasses) R1 ring links |
| `ble.gatt-status.<code>` | counter | Android GATT connection-state callbacks with a non-success status (e.g. 8, 133) |
| `ble.ios-disconnect-error.<code>` | counter | iOS disconnects with an error |
| `device.g2.used`, `device.g2.touchpad-used`, `device.r1.used`, `device.wear.reachable`, `device.wear.used` | counter (1 per day) | Which devices were in use that day |
| `device.g2.paired`, `device.r1.paired` | snapshot | Whether glasses and a ring are paired |
| `device.g2.preview-only` | snapshot | Whether the app is in phone-preview-only mode |
| `device.r1.connection` | snapshot | `glasses` or `direct` ring connection |
Full level adds:
| Name | Kind | Meaning |
| --- | --- | --- |
| `apikey.<kind>` | snapshot | Whether an API key or token of that kind is set (`anthropic`, `openai`, `elevenlabs`, `soniox`, `mapbox`, `roam`, `nightscout`, `agent-bridge`, `evenhub`), never the key |
| `config.nightscout-site` | snapshot | Whether a Nightscout site is set (not its URL) |
| `config.terminal-connections`, `config.t3code-environments` | snapshot | How many are configured |
| `config.ui-font` | snapshot | The UI font: a bundled font's name and size, `installed:<size>` for a user-installed one, or the bitmap face |
| `setting.<key>` | snapshot | The value of every setting chosen from fixed options (on/off and pick-one settings), by its storage key; free-text settings are never included |
Connection counts are kept by the shared Kotlin session core
(ConnectionCounters.kt) and drained into the app's store every few minutes.
+64
View File
@@ -0,0 +1,64 @@
# stats.faceclaw.org
Receives the opt-in analytics reports the phone app sends (format:
PROTOCOL.md) and stores them in Postgres. A small Node server with no
framework, behind nginx on the `cumulus` VPS.
src/server.ts HTTP server: POST /v1/report, GET /healthz
src/report.ts report validation
src/db.ts storing a report
schema.sql tables and views (idempotent, applied on every deploy)
deploy.sh deploy from this working tree
deploy/ systemd unit, nginx site, env and one-time setup script
Node runs the TypeScript directly (type stripping, Node 22.18 or later), so
there is no build step. `npm run check` type-checks; `npm test` runs the
tests. The database test needs a scratch database:
`STATS_TEST_DATABASE=<name> npm test`, with the other `PG*` variables
pointing at its server.
## Deploying
./deploy.sh
copies this directory to `~/faceclaw-stats` on the server, installs the
dependencies, applies `schema.sql` and restarts the service. The first
time, it stops after copying, and the one-time root setup must be run there:
ssh -t cumulus 'sudo bash ~/faceclaw-stats/deploy/setup-server.sh'
That creates a Postgres role named after the deploy user and the
`faceclaw_stats` database on the Postgres 16 cluster (port 5433; the older
14 cluster on 5432 is left alone), installs and starts
`faceclaw-stats.service`, adds the nginx site, and gets its certificate
with certbot. It is safe to rerun. The nginx site is only installed the first
time, since certbot edits it afterwards, so later changes to
`deploy/nginx-site.conf` have to be made on the server too.
The service runs as the deploy user with `Restart=always`, so `deploy.sh`
restarts it by ending the process, without needing root.
## Privacy
nginx keeps no access log for this site and the server stores only the
report body, so the database has no IP addresses. Reports carry a random
install id that the phone deletes when analytics is turned off.
## Looking at the data
ssh cumulus psql -p 5433 faceclaw_stats
Views:
- `daily_counters`: counter totals per day, split by version, platform and
build, with how many installs reported each.
- `latest_reports`: each install's most recent report.
- `latest_snapshot_values`: snapshot values across installs' latest reports.
For example, installs and connection failures per version over the last
week:
select app_version, name, sum(total) as total, max(installs) as installs
from daily_counters
where day > current_date - 7 and name like 'g2.failure.%'
group by 1, 2 order by 1, 3 desc;
+32
View File
@@ -0,0 +1,32 @@
#!/bin/bash
# Deploys the stats server from this working tree to the VPS: copies the
# sources to ~/faceclaw-stats, installs dependencies and, once the one-time
# root setup (deploy/setup-server.sh) has been done, applies schema.sql and
# restarts the service. Set FACECLAW_STATS_HOST to deploy somewhere other
# than the `cumulus` ssh host.
set -euo pipefail
cd "$(dirname "$0")"
HOST="${FACECLAW_STATS_HOST:-cumulus}"
DIR=faceclaw-stats
rsync -az --delete --exclude node_modules --exclude .DS_Store ./ "$HOST:$DIR/"
ssh "$HOST" bash -s -- "$DIR" "$HOST" <<'REMOTE'
set -euo pipefail
cd "$HOME/$1"
export NVM_DIR="$HOME/.nvm"
. "$NVM_DIR/nvm.sh" >/dev/null
nvm use 22 >/dev/null
npm ci --omit=dev --no-audit --no-fund --loglevel=error --no-update-notifier
if [ ! -f /etc/systemd/system/faceclaw-stats.service ]; then
echo "Copied. Now run the one-time setup: ssh -t $2 'sudo bash ~/$1/deploy/setup-server.sh'"
exit 0
fi
set -a; . deploy/stats.env; set +a
psql -v ON_ERROR_STOP=1 -q -f schema.sql
# The service runs as this user with Restart=always, so ending the process
# restarts it on the new code without needing root.
pkill -TERM -u "$USER" -f "$PWD/src/server.ts" || true
sleep 3
systemctl is-active faceclaw-stats
curl -fsS "http://127.0.0.1:$PORT/healthz" && echo
REMOTE
@@ -0,0 +1,21 @@
# Template: setup-server.sh fills in @USER@ and @DIR@ and installs it as
# /etc/systemd/system/faceclaw-stats.service.
[Unit]
Description=Faceclaw stats server (stats.faceclaw.org)
After=network.target postgresql.service
[Service]
Type=simple
User=@USER@
WorkingDirectory=@DIR@
EnvironmentFile=@DIR@/deploy/stats.env
ExecStart=@DIR@/deploy/run.sh
# deploy.sh restarts the server by ending the process.
Restart=always
RestartSec=2
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=full
[Install]
WantedBy=multi-user.target
+33
View File
@@ -0,0 +1,33 @@
# stats.faceclaw.org, installed by setup-server.sh as
# /etc/nginx/sites-available/stats.faceclaw.org. certbot then adds the TLS
# server block and the HTTP-to-HTTPS redirect to the installed copy, so
# setup-server.sh never overwrites it; make later changes on the server too.
limit_req_zone $binary_remote_addr zone=faceclaw_stats:1m rate=10r/m;
server {
listen 80;
server_name stats.faceclaw.org;
# Reports are anonymous: don't keep the IP addresses they come from.
access_log off;
limit_req_log_level notice;
client_max_body_size 64k;
location = /v1/report {
limit_req zone=faceclaw_stats burst=20 nodelay;
limit_req_status 429;
proxy_pass http://127.0.0.1:8290;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
location = /healthz {
proxy_pass http://127.0.0.1:8290;
}
location / {
default_type text/plain;
return 200 "Faceclaw's opt-in usage statistics endpoint. See https://faceclaw.org/privacy.html\n";
}
}
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
# Started by faceclaw-stats.service, with deploy/stats.env in the environment.
# Node comes from nvm, as for the other services on this host.
cd "$(dirname "$0")/.."
export NVM_DIR="$HOME/.nvm"
. "$NVM_DIR/nvm.sh"
nvm use 22 >/dev/null
# The absolute path lets deploy.sh find this process to restart it.
exec node "$PWD/src/server.ts"
+57
View File
@@ -0,0 +1,57 @@
#!/bin/bash
# One-time root setup for the stats server on the VPS; safe to rerun. Run it
# via sudo, as the user that owns the deployed copy (deploy.sh puts it there):
#
# ssh -t cumulus 'sudo bash ~/faceclaw-stats/deploy/setup-server.sh'
#
# It creates a Postgres role (named after that user, so the service and
# psql connect by peer auth) and the database on the cluster in stats.env,
# installs and starts the systemd service, adds the nginx site, and gets the
# site's Let's Encrypt certificate with certbot.
set -euo pipefail
[ "$(id -u)" = 0 ] || { echo "Run this with sudo." >&2; exit 1; }
DEPLOY_USER="${SUDO_USER:?Run this via sudo from the user that owns the deployed copy.}"
DIR="$(cd "$(dirname "$0")/.." && pwd)"
DOMAIN=stats.faceclaw.org
set -a; . "$DIR/deploy/stats.env"; set +a
cd /
echo "== Postgres: role $DEPLOY_USER, database $PGDATABASE (port $PGPORT)"
pg_lsclusters --no-header | awk -v port="$PGPORT" '$3 == port && $4 == "online" { found = 1 } END { exit !found }' \
|| { echo "No online Postgres cluster on port $PGPORT." >&2; exit 1; }
sudo -u postgres psql -p "$PGPORT" -v ON_ERROR_STOP=1 -q -c \
"do \$\$ begin if not exists (select from pg_roles where rolname = '$DEPLOY_USER') then create role \"$DEPLOY_USER\" login; end if; end \$\$;"
if ! sudo -u postgres psql -p "$PGPORT" -Atc "select 1 from pg_database where datname = '$PGDATABASE'" | grep -q 1; then
sudo -u postgres createdb -p "$PGPORT" -O "$DEPLOY_USER" "$PGDATABASE"
fi
sudo -u "$DEPLOY_USER" psql -h "$PGHOST" -p "$PGPORT" -d "$PGDATABASE" -v ON_ERROR_STOP=1 -q -f "$DIR/schema.sql"
echo "== systemd: faceclaw-stats.service"
[ -d "$DIR/node_modules" ] || { echo "No node_modules in $DIR; run deploy.sh first." >&2; exit 1; }
sed -e "s|@USER@|$DEPLOY_USER|g" -e "s|@DIR@|$DIR|g" "$DIR/deploy/faceclaw-stats.service" \
> /etc/systemd/system/faceclaw-stats.service
systemctl daemon-reload
systemctl enable faceclaw-stats.service
systemctl restart faceclaw-stats.service
echo "== nginx: $DOMAIN"
SITE="/etc/nginx/sites-available/$DOMAIN"
# certbot edits the installed copy, so only install it the first time.
[ -e "$SITE" ] || cp "$DIR/deploy/nginx-site.conf" "$SITE"
ln -sf "$SITE" "/etc/nginx/sites-enabled/$DOMAIN"
if ! nginx -t; then
# Don't leave a broken site enabled: it would block reloads for every site.
rm -f "/etc/nginx/sites-enabled/$DOMAIN"
echo "nginx rejected the site config; it has been disabled again." >&2
exit 1
fi
systemctl reload nginx
echo "== TLS certificate"
if [ ! -d "/etc/letsencrypt/live/$DOMAIN" ]; then
certbot --nginx -d "$DOMAIN" --non-interactive --redirect
fi
echo "== Checking https://$DOMAIN/healthz"
sleep 2
curl -fsS "https://$DOMAIN/healthz" && echo
+7
View File
@@ -0,0 +1,7 @@
# Server settings, read by the systemd unit (EnvironmentFile) and sourced by
# setup-server.sh and deploy.sh. The database is on the Postgres 16 cluster
# (port 5433); the server's older 14 cluster on 5432 is left alone.
PORT=8290
PGHOST=/var/run/postgresql
PGPORT=5433
PGDATABASE=faceclaw_stats
+211
View File
@@ -0,0 +1,211 @@
{
"name": "faceclaw-stats-server",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "faceclaw-stats-server",
"license": "GPL-3.0-or-later",
"dependencies": {
"pg": "^8.16.3"
},
"devDependencies": {
"@types/node": "^22.18.0",
"@types/pg": "^8.15.5",
"typescript": "~5.9.0"
},
"engines": {
"node": ">=22.18"
}
},
"node_modules/@types/node": {
"version": "22.20.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.5.tgz",
"integrity": "sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/@types/pg": {
"version": "8.23.1",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz",
"integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"pg-protocol": "*",
"pg-types": "^2.2.0"
}
},
"node_modules/pg": {
"version": "8.23.1",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz",
"integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==",
"license": "MIT",
"dependencies": {
"pg-connection-string": "^2.14.1",
"pg-pool": "^3.14.0",
"pg-protocol": "^1.16.1",
"pg-types": "2.2.0",
"pgpass": "1.0.5"
},
"engines": {
"node": ">= 16.0.0"
},
"optionalDependencies": {
"pg-cloudflare": "^1.4.1"
},
"peerDependencies": {
"pg-native": ">=3.0.1"
},
"peerDependenciesMeta": {
"pg-native": {
"optional": true
}
}
},
"node_modules/pg-cloudflare": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz",
"integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==",
"license": "MIT",
"optional": true
},
"node_modules/pg-connection-string": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz",
"integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==",
"license": "MIT"
},
"node_modules/pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
"license": "ISC",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/pg-pool": {
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
"license": "MIT",
"peerDependencies": {
"pg": ">=8.0"
}
},
"node_modules/pg-protocol": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz",
"integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==",
"license": "MIT"
},
"node_modules/pg-types": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
"license": "MIT",
"dependencies": {
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/pgpass": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
"license": "MIT",
"dependencies": {
"split2": "^4.1.0"
}
},
"node_modules/postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/postgres-bytea": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-date": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-interval": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"license": "MIT",
"dependencies": {
"xtend": "^4.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
"license": "ISC",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"license": "MIT",
"engines": {
"node": ">=0.4"
}
}
}
}
+23
View File
@@ -0,0 +1,23 @@
{
"name": "faceclaw-stats-server",
"private": true,
"description": "Receives Faceclaw's opt-in analytics reports at stats.faceclaw.org and stores them in Postgres.",
"license": "GPL-3.0-or-later",
"type": "module",
"engines": {
"node": ">=22.18"
},
"scripts": {
"start": "node src/server.ts",
"check": "tsc -p tsconfig.json",
"test": "node --test test/*.test.ts"
},
"dependencies": {
"pg": "^8.16.3"
},
"devDependencies": {
"@types/node": "^22.18.0",
"@types/pg": "^8.15.5",
"typescript": "~5.9.0"
}
}
+57
View File
@@ -0,0 +1,57 @@
-- Schema for the analytics reports (PROTOCOL.md). Idempotent: deploy.sh
-- applies it on every deploy, so changes must be written as additive
-- `if not exists` / `create or replace` statements.
create table if not exists reports (
report_id uuid primary key,
install_id uuid not null,
received_at timestamptz not null default now(),
created_at timestamptz not null,
schema_version integer not null,
level text not null,
app_version text not null,
platform text not null,
build text not null,
-- The validated report as received, for anything the tables below don't cover.
payload jsonb not null
);
create index if not exists reports_received_at_idx on reports (received_at);
create index if not exists reports_install_idx on reports (install_id, received_at);
create table if not exists report_counters (
report_id uuid not null references reports (report_id) on delete cascade,
day date not null,
name text not null,
value bigint not null,
primary key (report_id, day, name)
);
create index if not exists report_counters_day_name_idx on report_counters (day, name);
create table if not exists report_snapshot (
report_id uuid not null references reports (report_id) on delete cascade,
key text not null,
value jsonb not null,
primary key (report_id, key)
);
-- Counter totals per day, split by version, platform and build, with how
-- many installs reported each.
create or replace view daily_counters as
select r.app_version, r.platform, r.build, c.day, c.name,
sum(c.value) as total, count(distinct r.install_id) as installs
from report_counters c
join reports r using (report_id)
group by r.app_version, r.platform, r.build, c.day, c.name;
-- Each install's most recent report.
create or replace view latest_reports as
select distinct on (install_id) *
from reports
order by install_id, received_at desc;
-- Snapshot values from each install's most recent report, counted.
create or replace view latest_snapshot_values as
select r.app_version, r.platform, s.key, s.value, count(*) as installs
from latest_reports r
join report_snapshot s using (report_id)
group by r.app_version, r.platform, s.key, s.value;
+69
View File
@@ -0,0 +1,69 @@
import type { Pool } from "pg";
import type { Report } from "./report.ts";
/**
* Stores one report and its rows in a single transaction. The phone keeps
* resending a report until it sees a 2xx, so a report id that is already
* stored is a retry whose response was lost, not new data.
*/
export async function saveReport(pool: Pool, report: Report): Promise<"saved" | "duplicate"> {
const client = await pool.connect();
try {
await client.query("begin");
const inserted = await client.query(
`insert into reports
(report_id, install_id, created_at, schema_version, level, app_version, platform, build, payload)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
on conflict (report_id) do nothing`,
[
report.reportId,
report.installId,
report.createdAt,
report.schema,
report.level,
report.app.version,
report.app.platform,
report.app.build,
JSON.stringify(report),
],
);
if (inserted.rowCount === 0) {
await client.query("rollback");
return "duplicate";
}
const days: string[] = [];
const names: string[] = [];
const values: number[] = [];
for (const [day, counts] of Object.entries(report.counters)) {
for (const [name, value] of Object.entries(counts)) {
days.push(day);
names.push(name);
values.push(value);
}
}
if (names.length) {
await client.query(
`insert into report_counters (report_id, day, name, value)
select $1, day::date, name, value from unnest($2::text[], $3::text[], $4::bigint[]) as t(day, name, value)`,
[report.reportId, days, names, values],
);
}
const keys = Object.keys(report.snapshot);
if (keys.length) {
await client.query(
`insert into report_snapshot (report_id, key, value)
select $1, key, value from unnest($2::text[], $3::jsonb[]) as t(key, value)`,
[report.reportId, keys, keys.map((key) => JSON.stringify(report.snapshot[key]))],
);
}
await client.query("commit");
return "saved";
} catch (error) {
await client.query("rollback").catch(() => undefined);
throw error;
} finally {
client.release();
}
}
+126
View File
@@ -0,0 +1,126 @@
/**
* The analytics report format the phone app uploads (PROTOCOL.md), and its
* validation. The app-side producer is app/analytics/analytics-report.ts;
* keep the two in step.
*
* Everything is checked against tight limits so a malformed or hostile
* client can't store much: unknown fields are ignored, and anything
* out of range rejects the whole report.
*/
export const REPORT_SCHEMA_VERSION = 1;
export type ReportLevel = "minimal" | "full";
export type SnapshotValue = boolean | number | string;
export interface Report {
schema: number;
reportId: string;
installId: string;
level: ReportLevel;
/** When the phone sealed the report (ISO 8601). */
createdAt: string;
app: { version: string; platform: string; build: string };
/** UTC day (YYYY-MM-DD) → counter name → count. */
counters: Record<string, Record<string, number>>;
/** Point-in-time state (devices, API-key presence, settings), flat namespaced keys. */
snapshot: Record<string, SnapshotValue>;
}
export class ReportError extends Error {}
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const VERSION = /^[0-9A-Za-z.+-]{1,32}$/;
const TOKEN = /^[a-z0-9-]{1,24}$/;
const DAY = /^\d{4}-\d{2}-\d{2}$/;
const COUNTER_NAME = /^[a-z0-9][a-z0-9_.:-]{0,119}$/;
// Snapshot keys embed the app's settings keys, which are camelCase.
const SNAPSHOT_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,119}$/;
const MAX_DAYS = 100;
const MAX_COUNTERS = 2000;
const MAX_COUNT = 1_000_000_000;
const MAX_SNAPSHOT_KEYS = 500;
const MAX_SNAPSHOT_STRING = 200;
export function parseReport(value: unknown): Report {
const body = record(value, "report");
if (body.schema !== REPORT_SCHEMA_VERSION) throw new ReportError(`unsupported schema ${String(body.schema)}`);
const level = body.level;
if (level !== "minimal" && level !== "full") throw new ReportError("bad level");
const app = record(body.app, "app");
return {
schema: REPORT_SCHEMA_VERSION,
reportId: uuid(body.reportId, "reportId"),
installId: uuid(body.installId, "installId"),
level,
createdAt: timestamp(body.createdAt, "createdAt"),
app: {
version: matching(app.version, VERSION, "app.version"),
platform: matching(app.platform, TOKEN, "app.platform"),
build: matching(app.build, TOKEN, "app.build"),
},
counters: counters(body.counters ?? {}),
snapshot: snapshot(body.snapshot ?? {}),
};
}
function counters(value: unknown): Report["counters"] {
const days = record(value, "counters");
const dayKeys = Object.keys(days);
if (dayKeys.length > MAX_DAYS) throw new ReportError("too many days");
const result: Report["counters"] = {};
let total = 0;
for (const day of dayKeys) {
if (!DAY.test(day) || Number.isNaN(Date.parse(`${day}T00:00:00Z`))) throw new ReportError(`bad day ${day}`);
const entries = record(days[day], `counters.${day}`);
const counts: Record<string, number> = {};
for (const [name, count] of Object.entries(entries)) {
if (!COUNTER_NAME.test(name)) throw new ReportError(`bad counter name ${name}`);
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0 || count > MAX_COUNT) {
throw new ReportError(`bad count for ${name}`);
}
if (++total > MAX_COUNTERS) throw new ReportError("too many counters");
counts[name] = count;
}
result[day] = counts;
}
return result;
}
function snapshot(value: unknown): Report["snapshot"] {
const entries = Object.entries(record(value, "snapshot"));
if (entries.length > MAX_SNAPSHOT_KEYS) throw new ReportError("too many snapshot keys");
const result: Report["snapshot"] = {};
for (const [key, item] of entries) {
if (!SNAPSHOT_KEY.test(key)) throw new ReportError(`bad snapshot key ${key}`);
const ok =
typeof item === "boolean" ||
(typeof item === "number" && Number.isFinite(item)) ||
(typeof item === "string" && item.length <= MAX_SNAPSHOT_STRING);
if (!ok) throw new ReportError(`bad snapshot value for ${key}`);
result[key] = item;
}
return result;
}
function record(value: unknown, what: string): Record<string, unknown> {
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new ReportError(`${what} must be an object`);
return value as Record<string, unknown>;
}
function uuid(value: unknown, what: string): string {
const text = typeof value === "string" ? value.toLowerCase() : "";
if (!UUID.test(text)) throw new ReportError(`bad ${what}`);
return text;
}
function matching(value: unknown, pattern: RegExp, what: string): string {
if (typeof value !== "string" || !pattern.test(value)) throw new ReportError(`bad ${what}`);
return value;
}
function timestamp(value: unknown, what: string): string {
if (typeof value !== "string" || value.length > 40 || Number.isNaN(Date.parse(value))) throw new ReportError(`bad ${what}`);
return new Date(value).toISOString();
}
+91
View File
@@ -0,0 +1,91 @@
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { Pool } from "pg";
import { saveReport } from "./db.ts";
import { parseReport, ReportError } from "./report.ts";
/**
* Receives the phone app's daily analytics reports (PROTOCOL.md) and stores
* them in Postgres. Runs behind nginx, which terminates TLS and rate-limits;
* the database connection comes from the standard PG* environment variables.
*
* Nothing about the request itself is stored or logged: no IP address, no
* headers, only the report body.
*/
const PORT = Number(process.env.PORT ?? 8290);
const HOST = process.env.HOST ?? "127.0.0.1";
const MAX_BODY_BYTES = 64 * 1024;
const pool = new Pool({ max: 4 });
pool.on("error", (error) => console.error("idle database client failed:", error.message));
const server = createServer((request, response) => {
handle(request, response).catch((error: unknown) => {
console.error("request failed:", error instanceof Error ? error.message : error);
if (!response.headersSent) reply(response, 500, { error: "internal error" });
});
});
async function handle(request: IncomingMessage, response: ServerResponse): Promise<void> {
const path = (request.url ?? "/").split("?")[0];
if (path === "/v1/report") {
if (request.method !== "POST") return reply(response, 405, { error: "POST only" });
if (Number(request.headers["content-length"] ?? 0) > MAX_BODY_BYTES) {
response.setHeader("connection", "close");
return reply(response, 413, { error: "too large" });
}
const body = await readBody(request);
if (body === null) return reply(response, 413, { error: "too large" });
let report;
try {
report = parseReport(JSON.parse(body));
} catch (error) {
if (error instanceof SyntaxError || error instanceof ReportError) {
return reply(response, 400, { error: error.message });
}
throw error;
}
await saveReport(pool, report);
// Room for more later (e.g. the latest released version, for update checks).
return reply(response, 200, { ok: true });
}
if (path === "/healthz" && request.method === "GET") {
await pool.query("select 1");
return reply(response, 200, { ok: true });
}
reply(response, 404, { error: "not found" });
}
/**
* The request body as text, or null if it is over MAX_BODY_BYTES. An
* oversized body is still read to the end (and dropped) so the client gets
* the 413 instead of a reset connection; nginx caps it before it gets here.
*/
function readBody(request: IncomingMessage): Promise<string | null> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let size = 0;
request.on("data", (chunk: Buffer) => {
size += chunk.length;
if (size <= MAX_BODY_BYTES) chunks.push(chunk);
});
request.on("end", () => resolve(size > MAX_BODY_BYTES ? null : Buffer.concat(chunks).toString("utf8")));
request.on("error", reject);
});
}
function reply(response: ServerResponse, status: number, body: object): void {
response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" });
response.end(JSON.stringify(body));
}
function shutdown(): void {
server.close(() => {
void pool.end().finally(() => process.exit(0));
});
setTimeout(() => process.exit(0), 5000).unref();
}
process.on("SIGTERM", shutdown);
process.on("SIGINT", shutdown);
server.listen(PORT, HOST, () => console.log(`faceclaw-stats listening on ${HOST}:${PORT}`));
+43
View File
@@ -0,0 +1,43 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { test } from "node:test";
import { Pool } from "pg";
import { saveReport } from "../src/db.ts";
import { parseReport } from "../src/report.ts";
import { sampleReport } from "./sample.ts";
// Needs a scratch database: set STATS_TEST_DATABASE to its name (the other
// PG* variables pick the server). The tables are created if missing, and
// emptied before the test.
const database = process.env.STATS_TEST_DATABASE;
void test("stores a report once, with its counters and snapshot", { skip: !database && "STATS_TEST_DATABASE not set" }, async () => {
const pool = new Pool({ database });
try {
await pool.query(readFileSync(new URL("../schema.sql", import.meta.url), "utf8"));
await pool.query("truncate reports cascade");
const report = parseReport(sampleReport());
assert.equal(await saveReport(pool, report), "saved");
assert.equal(await saveReport(pool, report), "duplicate");
const counters = await pool.query(
"select day::text, name, value::int from report_counters order by day, name",
);
assert.deepEqual(counters.rows, [
{ day: "2026-10-09", name: "g2.connect", value: 3 },
{ day: "2026-10-09", name: "g2.disconnect", value: 2 },
{ day: "2026-10-10", name: "g2.connect", value: 1 },
]);
const snapshot = await pool.query("select key, value from report_snapshot order by key");
assert.deepEqual(snapshot.rows, [
{ key: "apikey.anthropic", value: false },
{ key: "device.g2.paired", value: true },
{ key: "setting.ui-font", value: "Roboto-Light.ttf:16" },
]);
const daily = await pool.query("select total::int, installs::int from daily_counters where name = 'g2.connect' and day = '2026-10-09'");
assert.deepEqual(daily.rows, [{ total: 3, installs: 1 }]);
} finally {
await pool.end();
}
});
+61
View File
@@ -0,0 +1,61 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { parseReport, ReportError } from "../src/report.ts";
import { sampleReport } from "./sample.ts";
void test("accepts a well-formed report and normalizes ids and timestamps", () => {
const report = parseReport(
sampleReport({ reportId: "0F8FAD5B-D9CB-469F-A165-70867728950E", createdAt: "2026-10-10T08:00:00-04:00" }),
);
assert.equal(report.reportId, "0f8fad5b-d9cb-469f-a165-70867728950e");
assert.equal(report.createdAt, "2026-10-10T12:00:00.000Z");
assert.deepEqual(report.counters["2026-10-09"], { "g2.connect": 3, "g2.disconnect": 2 });
assert.equal(report.snapshot["setting.ui-font"], "Roboto-Light.ttf:16");
});
void test("accepts camelCase snapshot keys but only lowercase counter names", () => {
assert.equal(parseReport(sampleReport({ snapshot: { "setting.display.timeFormat": "12h" } })).snapshot["setting.display.timeFormat"], "12h");
assert.throws(() => parseReport(sampleReport({ counters: { "2026-10-10": { "g2.Connect": 1 } } })), ReportError);
});
void test("ignores unknown fields", () => {
const report = parseReport(sampleReport({ extra: "x" }));
assert.equal("extra" in report, false);
});
void test("rejects malformed reports", () => {
const bad: Record<string, unknown>[] = [
{ schema: 2 },
{ level: "everything" },
{ reportId: "not-a-uuid" },
{ app: { version: "0.8.3; drop table", platform: "android", build: "official" } },
{ app: { version: "0.8.3", platform: "Android", build: "official" } },
{ createdAt: "yesterday" },
{ counters: { "2026-13-45": { a: 1 } } },
{ counters: { "2026-10-10": { "Bad Name": 1 } } },
{ counters: { "2026-10-10": { a: -1 } } },
{ counters: { "2026-10-10": { a: 1.5 } } },
{ counters: [] },
{ snapshot: { a: { nested: true } } },
{ snapshot: { "bad key": true } },
{ snapshot: { a: "x".repeat(201) } },
{ snapshot: { a: Number.NaN } },
];
for (const overrides of bad) {
assert.throws(() => parseReport(sampleReport(overrides)), ReportError, JSON.stringify(overrides));
}
assert.throws(() => parseReport(null), ReportError);
assert.throws(() => parseReport([]), ReportError);
});
void test("caps how many counters and snapshot keys a report may carry", () => {
const manyDays: Record<string, Record<string, number>> = {};
for (let day = 0; day < 101; day++) {
manyDays[new Date(Date.UTC(2026, 0, 1 + day)).toISOString().slice(0, 10)] = { a: 1 };
}
assert.throws(() => parseReport(sampleReport({ counters: manyDays })), /too many days/);
const manyKeys: Record<string, boolean> = {};
for (let index = 0; index < 501; index++) manyKeys[`k${index}`] = true;
assert.throws(() => parseReport(sampleReport({ snapshot: manyKeys })), /too many snapshot keys/);
});
+14
View File
@@ -0,0 +1,14 @@
/** A valid report body, with top-level fields replaced by `overrides`. */
export function sampleReport(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
schema: 1,
reportId: "0f8fad5b-d9cb-469f-a165-70867728950e",
installId: "7c9e6679-7425-40de-944b-e07fc1f90ae7",
level: "full",
createdAt: "2026-10-10T12:00:00.000Z",
app: { version: "0.8.3", platform: "android", build: "official" },
counters: { "2026-10-09": { "g2.connect": 3, "g2.disconnect": 2 }, "2026-10-10": { "g2.connect": 1 } },
snapshot: { "device.g2.paired": true, "apikey.anthropic": false, "setting.ui-font": "Roboto-Light.ttf:16" },
...overrides,
};
}
+17
View File
@@ -0,0 +1,17 @@
{
// Node runs the .ts sources directly (type stripping, Node >= 22.18), so
// this is for type checking only.
"compilerOptions": {
"target": "ES2023",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"strict": true,
"noEmit": true,
"allowImportingTsExtensions": true,
"erasableSyntaxOnly": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"types": ["node"]
},
"include": ["src/**/*", "test/**/*"]
}