diff --git a/.changeset/stats-cloud-operations.md b/.changeset/stats-cloud-operations.md
new file mode 100644
index 0000000..2868323
--- /dev/null
+++ b/.changeset/stats-cloud-operations.md
@@ -0,0 +1,5 @@
+---
+"ftw-webapp": patch
+---
+
+The private stats view shows how FTW Cloud runs: boxes and phones connected, the share on QUIC, data out this month against the free allowance, drops, push results, assistant calls, latency and each deploy's drain, from hourly sums over the whole service. It also shows the exact fleet and energy report totals for closed days. Nothing in it names a home, a device or an address.
diff --git a/stats/.dev.vars.example b/stats/.dev.vars.example
index 4e592c8..f52156c 100644
--- a/stats/.dev.vars.example
+++ b/stats/.dev.vars.example
@@ -4,3 +4,4 @@ CLOUDFLARE_ANALYTICS_TOKEN=replace_with_a_token_that_can_read_the_ftw_energy_zon
RELAY_INGEST_SECRET=replace_with_at_least_32_random_bytes
ACCESS_TEAM_DOMAIN=https://your-team.cloudflareaccess.com
ACCESS_AUD=replace_with_the_access_application_aud
+CLOUD_OPS_KEY=replace_with_the_key_whose_sha256_ftwcloudd_holds
diff --git a/stats/README.md b/stats/README.md
index 3576355..ea14155 100644
--- a/stats/README.md
+++ b/stats/README.md
@@ -68,6 +68,10 @@ npx wrangler secret put CLOUDFLARE_ANALYTICS_TOKEN --config stats/wrangler.jsonc
npx wrangler secret put RELAY_INGEST_SECRET --config stats/wrangler.jsonc
```
+A fourth, `CLOUD_OPS_KEY`, opens FTW Cloud's internal feed. FTW Cloud holds
+only its SHA-256; srcfl/ftw-cloud `deploy/README.md` ("Watching it") shows
+how to make both. Without the key the collector waits.
+
Set `CLOUDFLARE_ZONE_ID` as a Worker variable. Give the Cloudflare token
`Account > Account Analytics > Read` and make sure its resource scope includes
the `ftw.energy` zone. The hourly collection asks for Cloudflare's server-side
@@ -109,6 +113,8 @@ The hourly cleanup keeps only the data each dashboard window needs:
- 90 days of GitHub snapshots, relay snapshots, fleet daily totals and
collector results;
- only the latest thresholded Cloud daily-report view, removed if over 90 days old;
+- 90 days of FTW Cloud's service hours, and only the latest exact view of
+ closed fleet and energy report days, removed if over 90 days old;
- 30 UTC days of GitHub traffic, referrers and popular paths;
- the last 7 complete UTC days of site traffic.
@@ -158,3 +164,38 @@ check-ins, never unique users or installations. The legacy signed ingest
route remains for compatibility and is not the Cloud usage source. GitHub
charts keep saved subtotals with repo coverage; star lines break when the
set changes, so adding repositories cannot look like organic growth.
+
+## FTW Cloud operations (private view)
+
+Sourceful runs FTW Cloud and the fleet from the private view. At minute 23
+each hour the Worker reads `https://cloud.ftw.energy/v1/ops` with
+`CLOUD_OPS_KEY` and keeps what it gets in D1:
+
+- **Service hours** (`cloud_ops_hours`, 90 days). Each hour sums over the
+ whole service: how much each counter rose, each gauge's low, mean and
+ high, each latency's p50, p95 and p99, how much of the hour a process ran
+ and how many started. The view shows boxes and phones connected, the share
+ on QUIC, data written this month against AWS's 100 GB of free data out,
+ drops by reason, push results, assistant calls by outcome, the slowest
+ p99 of the day, and each deploy's drain and the box handshakes that
+ followed.
+- **Exact daily reports** (`cloud_ops_days`, the latest view only). The
+ fleet and energy report totals as FTW Cloud holds them, below the public
+ group size of 5 too, but only for closed days: a fleet day once it ends,
+ an energy date four days after. Two pulls never differ by one report.
+ Fleet days from `ftw.fleet/2` reports also count how the boxes ran: the
+ slowest plan and fallback plans of each box's last day, the share of
+ failed driver reads, and the newest update of its last week, each a few
+ fixed words. The public view does not show these.
+
+Each pull asks from an hour before the newest hour kept, so a restart's late
+part of an hour replaces the row. A Cloud without a key answers 404, which
+the collector records as waiting. The parser takes only the fields
+`ftw.cloud-ops/1` names, at every level, and refuses the whole answer
+otherwise.
+
+None of it names a home, a device or an address. The limit, plainly: while
+the fleet is a handful of homes, a small count can still describe one of
+them to someone who knows them. Only people Access lets in see it.
+srcfl/ftw-cloud `privacy/internal-operations.md` has the full account.
+Per-home data stays in support access, opted in per ticket.
diff --git a/stats/migrations/0005_cloud_ops.sql b/stats/migrations/0005_cloud_ops.sql
new file mode 100644
index 0000000..417c156
--- /dev/null
+++ b/stats/migrations/0005_cloud_ops.sql
@@ -0,0 +1,20 @@
+-- FTW Cloud's own hours: sums over the whole service, never a home, device or address.
+CREATE TABLE cloud_ops_hours (
+ shard TEXT NOT NULL,
+ hour TEXT NOT NULL,
+ seconds INTEGER NOT NULL,
+ starts INTEGER NOT NULL,
+ counters_json TEXT NOT NULL,
+ gauges_json TEXT NOT NULL,
+ latency_json TEXT NOT NULL,
+ observed_at TEXT NOT NULL,
+ PRIMARY KEY (shard, hour)
+);
+-- The latest exact totals of closed fleet and energy report days, for the
+-- private view; never a report or an id.
+CREATE TABLE cloud_ops_days (
+ id INTEGER PRIMARY KEY CHECK(id=1),
+ observed_at TEXT NOT NULL,
+ fleet_json TEXT NOT NULL,
+ energy_json TEXT NOT NULL
+);
diff --git a/stats/src/dashboard.ts b/stats/src/dashboard.ts
index b46d9cf..014d68c 100644
--- a/stats/src/dashboard.ts
+++ b/stats/src/dashboard.ts
@@ -161,7 +161,7 @@ export function dashboardDocument(privateView: boolean): DashboardDocument {
-
+${privateView ? opsSection : ''}
@@ -406,7 +406,7 @@ export function dashboardDocument(privateView: boolean): DashboardDocument {
const height = value === null ? 0 : Math.max(1, Math.ceil((value / max) * 20));
bar.classList.add('h' + height);
bar.classList.toggle('withheld', value === null);
- wrap.title = shortDate(point.date) + ': ' + (value === null ? 'below public threshold' : formatter(value)) + (point.coverage ? ' · ' + point.coverage : '');
+ wrap.title = shortDate(point.date) + ': ' + (value === null ? point.gap || 'below public threshold' : formatter(value)) + (point.coverage ? ' · ' + point.coverage : '');
wrap.append(bar, node('span', points.length <= 16 ? dayLabel(point.date) : ''));
host.append(wrap);
}
@@ -522,7 +522,7 @@ export function dashboardDocument(privateView: boolean): DashboardDocument {
renderBars('relay-chart', (relay.series || []).slice(-48).map((point) => ({ date: point.observed_at, value: point.rooms })), number);
}
- function renderPrivate(data) {
+${privateView ? opsScript : ''} function renderPrivate(data) {
$('private-grid').hidden = false;
const refs = $('referrers');
refs.replaceChildren();
@@ -559,6 +559,7 @@ export function dashboardDocument(privateView: boolean): DashboardDocument {
renderSite(data.site);
renderFleet(data.fleet, data.mode === 'private');
if (data.mode === 'private') renderRelay(data.relay);
+ ${privateView ? "if (data.mode === 'private') renderOps(data.cloud_ops);" : ''}
if (data.mode === 'private') renderPrivate(data);
} catch (error) {
const notice = $('notice');
@@ -594,3 +595,123 @@ const styles = `
@media(max-width:560px){.mini-grid.site-grid{grid-template-columns:1fr}}
@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important}}
`
+
+// The private view's Service operations panel and its script. The public
+// page carries neither: its markup, copy and code stay as they were.
+const opsSection = `
+ FTW CLOUD · PRIVATE
Service operations
not collected
+ Each hour sums over the whole service. No home, device or address reaches this page. Cloud and this page keep the hours for 90 days.
+
+
Boxes connected—
+
Phones connected—
+
Over QUIC—boxes, last hour
+
Data out this month—of 100 GB free
+
+
+
+
+
+
+
+
Deploys and drains, 30 days
+
+
+
+
+
+
+ | Energy reports, exact | Reports | Solar kWh | Import / export kWh | Battery in / out kWh | Vehicle kWh |
+
+
+
+
+
+
+`
+
+const opsScript = ` function listInto(id, rows, empty) {
+ const host = $(id);
+ host.replaceChildren();
+ for (const [label, value] of rows) {
+ const item = node('li');
+ item.append(node('span', label), node('strong', value));
+ host.append(item);
+ }
+ if (!host.children.length) host.append(node('li', empty, 'empty'));
+ }
+ function ms(value) {
+ if (value === 'over_10s') return 'over 10 s';
+ return typeof value === 'number' ? (value < 10 ? value.toFixed(2) : fmt.format(value)) + ' ms' : '—';
+ }
+ function renderOps(ops) {
+ if (!ops) return;
+ $('ops-section').hidden = false;
+ const fleet = ops.fleet || { days: [] };
+ const days = (fleet.days || []).slice(-14).reverse();
+ listInto('ops-fleet', days.map((day) => [day.date, number(day.reports)]), 'No daily reports yet.');
+ if (fleet.state === 'stale') setText('ops-fleet-note', 'Source is stale: the last saved daily totals came ' + age(fleet.observed_at) + '. These are the last saved values.');
+ const labels = $('ops-fleet-labels');
+ labels.replaceChildren();
+ const titles = { ftw_versions: 'FTW versions', drivers: 'Device integrations', channels: 'Channels', battery_kwh: 'Battery', price_zones: 'Price zones', install_age: 'Install age',
+ plan_solve: 'Slowest plan', plan_fallbacks: 'Fallback plans', driver_errors: 'Failed driver reads', update: 'Newest update' };
+ if (days[0]) {
+ for (const key of Object.keys(titles).filter((k) => days[0][k])) {
+ const block = node('div', undefined, 'dimension');
+ block.append(node('h3', titles[key] + ', ' + shortDate(days[0].date)));
+ const chips = node('div', undefined, 'chips');
+ for (const [label, count] of Object.entries(days[0][key] || {}).sort((a, b) => b[1] - a[1])) chips.append(node('span', dimensionLabel(key, label) + ' · ' + number(count), 'chip'));
+ block.append(chips);
+ labels.append(block);
+ }
+ }
+ const energy = $('ops-energy');
+ energy.replaceChildren();
+ for (const day of ((ops.energy && ops.energy.days) || []).slice(-14).reverse()) {
+ const row = node('tr');
+ row.append(node('td', day.date));
+ for (const value of [number(day.reports), number(day.pv_generation_kwh), number(day.grid_import_kwh) + ' / ' + number(day.grid_export_kwh),
+ number(day.battery_charge_kwh) + ' / ' + number(day.battery_discharge_kwh), number(day.vehicle_charge_kwh)]) row.append(node('td', value, 'numeric'));
+ energy.append(row);
+ }
+ if (!energy.children.length) {
+ const row = node('tr');
+ const cell = node('td', 'No closed energy dates yet.', 'empty');
+ cell.colSpan = 6;
+ row.append(cell);
+ energy.append(row);
+ }
+ if (ops.state === 'empty') {
+ setText('ops-age', 'not collected');
+ renderBars('ops-chart', [], number);
+ for (const id of ['ops-drops', 'ops-push', 'ops-mcp', 'ops-latency', 'ops-deploys']) listInto(id, [], 'No service hours yet.');
+ return;
+ }
+ setText('ops-age', (ops.state === 'stale' ? 'stale · ' : '') + 'hour of ' + new Intl.DateTimeFormat(undefined, { day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit' }).format(new Date(ops.latest_hour)));
+ const now = ops.now || {};
+ setText('ops-boxes', now.boxes ? number(now.boxes.max) : '—');
+ setText('ops-boxes-note', now.boxes ? 'mean ' + number(now.boxes.mean) + ', low ' + number(now.boxes.min) + ' in the hour' : '');
+ setText('ops-phones', now.phones ? number(now.phones.max) : '—');
+ setText('ops-phones-note', now.phones ? 'mean ' + number(now.phones.mean) + ' in the hour' : '');
+ setText('ops-quic', typeof now.box_quic_percent === 'number' ? now.box_quic_percent + ' %' : '—');
+ setText('ops-quic-note', 'boxes; phones ' + (typeof now.app_quic_percent === 'number' ? now.app_quic_percent + ' %' : '—'));
+ const month = ops.month || {};
+ setText('ops-bytes', bytes(month.bytes_out));
+ setText('ops-budget', (typeof month.budget_percent === 'number' ? month.budget_percent + ' % ' : '') + 'of ' + bytes(month.budget_bytes) + ' free; tunnel ' + bytes(month.tun_bytes) +
+ '; ' + number(month.hours_recorded) + ' of ' + number(month.hours_so_far) + ' hours recorded');
+ renderBars('ops-chart', (ops.hours || []).map((h) => ({ date: h.hour, value: h.missing ? null : h.boxes_max,
+ gap: 'no hour from Cloud', coverage: new Date(h.hour).toISOString().slice(11, 16) + ' UTC' + (h.starts ? ' · restart in the hour' : '') })), number);
+ const week = ops.week || {};
+ // A week without hours is a week without data, not a quiet week.
+ const quiet = (text) => week.hours_recorded ? text : 'No service hours in the last 7 days.';
+ listInto('ops-drops', (week.drops || []).slice(0, 10).map((d) => [d.name.replace(/_/g, ' '), number(d.count)]), quiet('Nothing dropped.'));
+ const push = week.push || {};
+ listInto('ops-push', [['Alerts', push.alert], ['Widgets', push.widgets], ['Browsers', push.browser]].filter((r) => r[1] && r[1].sent)
+ .map(([label, r]) => [label + ' · ' + number(r.sent), r.ok_percent + ' % ok']).concat(push.breaker_opened ? [['Breaker opened', number(push.breaker_opened)]] : []), quiet('No pushes sent.'));
+ listInto('ops-mcp', ((week.mcp && week.mcp.outcomes) || []).map((o) => [o.name.replace(/_/g, ' '), number(o.count)])
+ .concat(week.mcp && week.mcp.p99_ms_max !== null && week.mcp.p99_ms_max !== undefined ? [['slowest hour p99', ms(week.mcp.p99_ms_max)]] : []), quiet('No assistant calls.'));
+ listInto('ops-latency', (ops.latency_24h || []).filter((l) => l.n > 0).map((l) => [l.name.replace(/_seconds/, '').replace(/_/g, ' ') + ' · ' + number(l.n), ms(l.p99_ms_max)]), 'Nothing timed.');
+ listInto('ops-deploys', (ops.deploys || []).slice(0, 8).map((d) => [shortDate(d.hour) + ' ' + new Date(d.hour).toISOString().slice(11, 16) + ' UTC',
+ (d.starts ? number(d.starts) + ' start' : 'no start') + ' · ' + number(d.drained_sockets) + ' drained' + (d.left_open ? ', ' + number(d.left_open) + ' left open' : '') + ' · ' + number(d.box_handshakes) + ' box handshakes']), 'No deploys in 30 days.');
+ }
+
+`
diff --git a/stats/src/index.ts b/stats/src/index.ts
index 3405b1d..c0052ae 100644
--- a/stats/src/index.ts
+++ b/stats/src/index.ts
@@ -2,6 +2,7 @@ import { accessIdentity } from './access.ts'
import { dashboardDocument } from './dashboard.ts'
import { collectEnergy } from './energy.ts'
import { collectFleet } from './fleet.ts'
+import { collectOps } from './ops.ts'
import { collectUsage } from './usage.ts'
import { collectGitHub } from './github.ts'
import { ingestRelay } from './relay.ts'
@@ -88,6 +89,7 @@ export default {
try { await collectUsage(env) } catch { failures.push('cloud:usage') }
try { await collectFleet(env) } catch { failures.push('cloud:fleet') }
try { await collectEnergy(env) } catch { failures.push('cloud:energy') }
+ try { await collectOps(env) } catch { failures.push('cloud:ops') }
} else { throw new Error('unknown collection schedule') }
await pruneStoredData(env.DB, controller.scheduledTime)
if (failures.length > 0) throw new Error('one or more scheduled collectors failed')
diff --git a/stats/src/ops.ts b/stats/src/ops.ts
new file mode 100644
index 0000000..19204fb
--- /dev/null
+++ b/stats/src/ops.ts
@@ -0,0 +1,327 @@
+import { saveCollectorRun } from './collector.ts'
+import { energyFields } from './energy.ts'
+import type { AppEnv, FleetDay } from './types.ts'
+
+// FTW Cloud's internal feed: the service's own hours, each a sum over the
+// whole service, and the exact totals of the anonymous fleet and energy
+// reports for closed days. Private view only. The parser takes exactly these
+// fields, so a field that could name a home, a device or an address cannot
+// ride in.
+
+const HOUR_MS = 3600_000
+const DAY_MS = 24 * HOUR_MS
+export const OPS_URL = 'https://cloud.ftw.energy/v1/ops'
+export const OPS_KEPT_DAYS = 90
+const MAX_HOURS = 168
+const MAX_BYTES = 1024 * 1024
+const MEANING = 'sums over the whole service, and closed daily totals over all reports; no home, device or address'
+// A box names one of the last three closed days, so the newest closed
+// energy date is four days back.
+const ENERGY_OPEN_DAYS = 4
+// AWS's free data out, across the account, a month.
+export const EGRESS_BUDGET_BYTES = 100e9
+
+const dimensions = ['ftw_versions', 'channels', 'drivers', 'battery_kwh', 'price_zones', 'install_age'] as const
+// How the boxes ran, from ftw.fleet/2 reports only: each a few fixed words.
+// A day without such a report has none of these fields.
+export const healthBuckets = {
+ plan_solve: ['none', '0-1s', '1-5s', '5s+'],
+ plan_fallbacks: ['none', '0', '1-3', '4+'],
+ driver_errors: ['none', '0', '0-1%', '1%+'],
+ update: ['none', 'ok', 'failed'],
+} as const
+const health = Object.keys(healthBuckets) as (keyof typeof healthBuckets)[]
+
+export interface Gauge { min: number; mean: number; max: number }
+export interface Latency { n: number; p50: number | null; p95: number | null; p99: number | null }
+export interface OpsHour {
+ hour: string; seconds: number; starts: number
+ counters: Record; gauges: Record; latency_ms: Record
+}
+export type ExactEnergyDay = { date: string; reports: number } & Record
+export interface OpsFeed { schema: 'ftw.cloud-ops/1'; meaning: string; shard: string; observed_at: string; hours: OpsHour[]; fleet_days: FleetDay[]; energy_days: ExactEnergyDay[] }
+
+function record(v: unknown, what: string): Record {
+ if (!v || typeof v !== 'object' || Array.isArray(v)) throw new Error('invalid ops ' + what)
+ return v as Record
+}
+function exact(v: Record, keys: readonly string[], what: string) {
+ if (Object.keys(v).sort().join() !== [...keys].sort().join()) throw new Error('unexpected ops fields in ' + what)
+}
+const int = (n: unknown, lo: number, hi: number) => typeof n === 'number' && Number.isSafeInteger(n) && n >= lo && n <= hi
+const name = (k: string) => /^[a-z0-9_]{1,80}$/.test(k)
+function series(v: unknown, what: string, each: (x: unknown) => T): Record {
+ const m = record(v, what)
+ if (Object.keys(m).length > 256) throw new Error('too many ops series')
+ for (const [k, x] of Object.entries(m)) { if (!name(k)) throw new Error('invalid ops series name'); each(x) }
+ return m as Record
+}
+function quantile(v: unknown) {
+ if (v !== null && (typeof v !== 'number' || !Number.isFinite(v) || v < 0 || v > 10_000)) throw new Error('invalid ops quantile')
+ return v as number | null
+}
+
+export function parseOps(value: unknown, nowMs: number): OpsFeed {
+ const v = record(value, 'feed')
+ exact(v, ['schema', 'meaning', 'shard', 'observed_at', 'hours', 'fleet_days', 'energy_days'], 'feed')
+ const observed = typeof v['observed_at'] === 'string' ? Date.parse(v['observed_at']) : NaN
+ if (v['schema'] !== 'ftw.cloud-ops/1' || v['meaning'] !== MEANING || typeof v['shard'] !== 'string' || !/^[a-z0-9-]{1,16}$/.test(v['shard'])) throw new Error('invalid ops feed')
+ if (!Number.isFinite(observed) || observed > nowMs + 60_000 || nowMs - observed > 2 * HOUR_MS) throw new Error('invalid ops time')
+ if (!Array.isArray(v['hours']) || v['hours'].length > MAX_HOURS || !Array.isArray(v['fleet_days']) || v['fleet_days'].length > 90 ||
+ !Array.isArray(v['energy_days']) || v['energy_days'].length > 31) throw new Error('invalid ops window')
+ let last = -Infinity
+ for (const item of v['hours']) {
+ const h = record(item, 'hour')
+ exact(h, ['hour', 'seconds', 'starts', 'counters', 'gauges', 'latency_ms'], 'hour')
+ const at = typeof h['hour'] === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:00:00Z$/.test(h['hour']) ? Date.parse(h['hour']) : NaN
+ if (!Number.isFinite(at) || at <= last || at + HOUR_MS > observed || at < nowMs - (OPS_KEPT_DAYS + 1) * DAY_MS) throw new Error('invalid ops hour')
+ last = at
+ if (!int(h['seconds'], 0, 7200) || !int(h['starts'], 0, 1000)) throw new Error('invalid ops coverage')
+ series(h['counters'], 'counters', (n) => { if (!int(n, 0, 1e15)) throw new Error('invalid ops counter') })
+ series(h['gauges'], 'gauges', (x) => {
+ const g = record(x, 'gauge')
+ exact(g, ['min', 'mean', 'max'], 'gauge')
+ const mean = g['mean'] as number
+ if (!int(g['min'], -1e15, 1e15) || !int(g['max'], -1e15, 1e15) || typeof mean !== 'number' || !Number.isFinite(mean) ||
+ (g['min'] as number) > (g['max'] as number) || mean < (g['min'] as number) - 1e-6 || mean > (g['max'] as number) + 1e-6) throw new Error('invalid ops gauge')
+ })
+ series(h['latency_ms'], 'latency', (x) => {
+ const l = record(x, 'latency')
+ exact(l, ['n', 'p50', 'p95', 'p99'], 'latency')
+ if (!int(l['n'], 0, 1e15)) throw new Error('invalid ops latency count')
+ const q = [quantile(l['p50']), quantile(l['p95']), quantile(l['p99'])]
+ if (l['n'] === 0 && q.some(p => p !== null)) throw new Error('quantiles without a count')
+ const known = q.filter((p): p is number => p !== null)
+ if (q.indexOf(null) !== -1 && q.slice(q.indexOf(null)).some(p => p !== null) || known.some((p, i) => i > 0 && p < known[i - 1]!)) throw new Error('quantiles out of order')
+ })
+ }
+ // A day shows only once closed, so two pulls never differ by one report.
+ const today = new Date(observed).toISOString().slice(0, 10)
+ const day = (date: string, newest: string, seen: Set) => {
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(Date.parse(date)) || new Date(date).toISOString().slice(0, 10) !== date ||
+ date >= newest || Date.parse(date) < nowMs - 91 * DAY_MS || seen.has(date)) throw new Error('invalid ops day')
+ seen.add(date)
+ }
+ const seen = new Set()
+ for (const item of v['fleet_days']) {
+ const d = record(item, 'day'), date = String(d['date'])
+ const withHealth = 'plan_solve' in d
+ exact(d, ['date', 'reports', ...dimensions, ...(withHealth ? health : [])], 'day')
+ day(date, today, seen)
+ const reports = d['reports']
+ if (!int(reports, 0, 100_000)) throw new Error('invalid ops fleet count')
+ for (const key of dimensions) {
+ const labels = record(d[key], 'fleet labels')
+ if (Object.keys(labels).length > 128) throw new Error('too many ops fleet labels')
+ for (const [label, n] of Object.entries(labels)) {
+ if (!label || label.length > 128 || /[\x00-\x1f\x7f]/.test(label) || !int(n, 1, reports as number)) throw new Error('invalid ops fleet label')
+ }
+ }
+ if (withHealth) for (const key of health) {
+ const labels = record(d[key], 'fleet health')
+ let total = 0
+ for (const [label, n] of Object.entries(labels)) {
+ if (!(healthBuckets[key] as readonly string[]).includes(label) || !int(n, 1, reports as number)) throw new Error('invalid ops fleet health')
+ total += n as number
+ }
+ if (total < 1 || total > (reports as number)) throw new Error('invalid ops fleet health total')
+ }
+ }
+ const closed = new Date(Date.parse(today) - (ENERGY_OPEN_DAYS - 1) * DAY_MS).toISOString().slice(0, 10)
+ const energySeen = new Set()
+ for (const item of v['energy_days']) {
+ const d = record(item, 'energy day')
+ exact(d, ['date', 'reports', ...energyFields], 'energy day')
+ day(String(d['date']), closed, energySeen)
+ if (!int(d['reports'], 1, 100_000) || energyFields.some(k => !int(d[k], 0, (d['reports'] as number) * 1000))) throw new Error('invalid ops energy day')
+ }
+ return v as unknown as OpsFeed
+}
+
+async function readCapped(r: Response, cap: number): Promise {
+ const reader = r.body?.getReader()
+ if (!reader) throw new Error('empty ops body')
+ const parts: Uint8Array[] = []
+ let size = 0
+ try {
+ while (true) {
+ const { done, value } = await reader.read()
+ if (done) break
+ size += value.length
+ if (size > cap) throw new Error('large ops body')
+ parts.push(value)
+ }
+ } finally { await reader.cancel() }
+ const bytes = new Uint8Array(size)
+ let offset = 0
+ for (const p of parts) { bytes.set(p, offset); offset += p.length }
+ return JSON.parse(new TextDecoder().decode(bytes))
+}
+
+// collectOps pulls the hours after the newest one kept, the last again in
+// case a restart added to it, and the exact day totals. Without a key it
+// waits; so does a Cloud that has none set (404).
+export async function collectOps(env: AppEnv, nowMs = Date.now()): Promise {
+ const started = new Date(nowMs).toISOString()
+ if (!env.CLOUD_OPS_KEY) { await saveCollectorRun(env.DB, 'cloud:ops', started, true, 'awaiting key'); return }
+ try {
+ const newest = await env.DB.prepare('SELECT MAX(hour) AS hour FROM cloud_ops_hours').first<{ hour: string | null }>()
+ const floor = Math.floor((nowMs - OPS_KEPT_DAYS * DAY_MS) / HOUR_MS) * HOUR_MS
+ const since = Math.max(floor, newest?.hour ? Date.parse(newest.hour) - HOUR_MS : floor)
+ const url = OPS_URL + '?since=' + encodeURIComponent(new Date(since).toISOString().replace('.000Z', 'Z'))
+ const r = await fetch(url, { redirect: 'manual', headers: { accept: 'application/json', authorization: 'Bearer ' + env.CLOUD_OPS_KEY }, signal: AbortSignal.timeout(15_000) })
+ if (r.status === 404) { await r.body?.cancel(); await saveCollectorRun(env.DB, 'cloud:ops', started, true, 'awaiting key on Cloud'); return }
+ if (!r.ok) { await r.body?.cancel(); throw new Error('ops source unavailable') }
+ const feed = parseOps(await readCapped(r, MAX_BYTES), nowMs)
+ const statements = feed.hours.map(h => env.DB.prepare(
+ `INSERT INTO cloud_ops_hours(shard,hour,seconds,starts,counters_json,gauges_json,latency_json,observed_at) VALUES(?,?,?,?,?,?,?,?)
+ ON CONFLICT(shard,hour) DO UPDATE SET seconds=excluded.seconds,starts=excluded.starts,counters_json=excluded.counters_json,
+ gauges_json=excluded.gauges_json,latency_json=excluded.latency_json,observed_at=excluded.observed_at`
+ ).bind(feed.shard, new Date(h.hour).toISOString(), h.seconds, h.starts, JSON.stringify(h.counters), JSON.stringify(h.gauges), JSON.stringify(h.latency_ms), feed.observed_at))
+ statements.push(env.DB.prepare(
+ 'INSERT INTO cloud_ops_days(id,observed_at,fleet_json,energy_json) VALUES(1,?,?,?) ON CONFLICT(id) DO UPDATE SET observed_at=excluded.observed_at,fleet_json=excluded.fleet_json,energy_json=excluded.energy_json'
+ ).bind(feed.observed_at, JSON.stringify(feed.fleet_days), JSON.stringify(feed.energy_days)))
+ await env.DB.batch(statements)
+ await saveCollectorRun(env.DB, 'cloud:ops', started, true, feed.hours.length + ' service hours')
+ } catch (e) { await saveCollectorRun(env.DB, 'cloud:ops', started, false, 'ops source unavailable or invalid'); throw e }
+}
+
+interface HourRow { shard: string; hour: string; seconds: number; starts: number; counters_json: string; gauges_json: string; latency_json: string }
+interface Hour { hour: string; seconds: number; starts: number; counters: Record; gauges: Record; latency: Record }
+
+// A quantile past ftwcloudd's last bucket (10 s) comes as null beside a
+// count. It is the slowest case there is, never a missing one.
+export const OVER_LAST_BUCKET = 'over_10s' as const
+type Slowest = number | typeof OVER_LAST_BUCKET | null
+function quantileOf(l: Latency | undefined, q: 'p50' | 'p95' | 'p99'): Slowest {
+ if (!l || l.n === 0) return null
+ return l[q] ?? OVER_LAST_BUCKET
+}
+function slowest(values: Slowest[]): Slowest {
+ if (values.includes(OVER_LAST_BUCKET)) return OVER_LAST_BUCKET
+ const v = values.filter((x): x is number => typeof x === 'number')
+ return v.length ? Math.max(...v) : null
+}
+const slowestOf = (rows: Hour[], key: string, q: 'p50' | 'p99' = 'p99') => slowest(rows.map(r => quantileOf(r.latency[key], q)))
+
+// byHour adds the shards of each hour together: each holds its own homes, so
+// counts and gauges sum, and a latency keeps the slowest shard's quantile.
+function byHour(rows: HourRow[]): Hour[] {
+ const hours = new Map()
+ for (const r of rows) {
+ const counters = JSON.parse(r.counters_json) as Record
+ const gauges = JSON.parse(r.gauges_json) as Record
+ const latency = JSON.parse(r.latency_json) as Record
+ const h = hours.get(r.hour)
+ if (!h) { hours.set(r.hour, { hour: r.hour, seconds: r.seconds, starts: r.starts, counters, gauges, latency }); continue }
+ h.seconds = Math.min(h.seconds, r.seconds)
+ h.starts += r.starts
+ for (const [k, n] of Object.entries(counters)) h.counters[k] = (h.counters[k] ?? 0) + n
+ for (const [k, g] of Object.entries(gauges)) {
+ const o = h.gauges[k]
+ h.gauges[k] = o ? { min: o.min + g.min, mean: o.mean + g.mean, max: o.max + g.max } : g
+ }
+ for (const [k, l] of Object.entries(latency)) {
+ const o = h.latency[k]
+ if (!o) { h.latency[k] = l; continue }
+ // A quantile of both is no slower than the slower shard's.
+ const merged = (q: 'p50' | 'p95' | 'p99') => { const w = slowest([quantileOf(o, q), quantileOf(l, q)]); return w === OVER_LAST_BUCKET ? null : w }
+ h.latency[k] = { n: o.n + l.n, p50: merged('p50'), p95: merged('p95'), p99: merged('p99') }
+ }
+ }
+ return [...hours.values()].sort((a, b) => a.hour.localeCompare(b.hour))
+}
+
+const sumOf = (rows: Record[], key: string) => rows.reduce((n, c) => n + (c[key] ?? 0), 0)
+function prefixed(totals: Record, prefix: string): { name: string; count: number }[] {
+ return Object.entries(totals).filter(([k, n]) => k.startsWith(prefix) && n > 0).map(([k, n]) => ({ name: k.slice(prefix.length), count: n })).sort((a, b) => b.count - a.count)
+}
+function share(a: number | undefined, b: number | undefined): number | null {
+ const total = (a ?? 0) + (b ?? 0)
+ return total > 0 ? Math.round((a ?? 0) / total * 1000) / 10 : null
+}
+
+// opsSummary is what the private view shows: the service now, the last two
+// days hour by hour with missing hours as gaps, a week of drops, pushes and
+// assistant calls, deploys over 30 days, data out this month against the
+// free allowance, and the fleet's exact days. Old data reads as stale, never
+// as nothing happening.
+export async function opsSummary(db: D1Database, nowMs: number): Promise> {
+ const monthStartMs = Date.UTC(new Date(nowMs).getUTCFullYear(), new Date(nowMs).getUTCMonth(), 1)
+ const monthStart = new Date(monthStartMs).toISOString()
+ const since = new Date(Math.min(nowMs - 30 * DAY_MS, monthStartMs)).toISOString()
+ const [stored, days] = await Promise.all([
+ db.prepare('SELECT shard,hour,seconds,starts,counters_json,gauges_json,latency_json FROM cloud_ops_hours WHERE hour >= ? ORDER BY hour ASC').bind(since).all(),
+ db.prepare('SELECT observed_at, fleet_json, energy_json FROM cloud_ops_days WHERE id=1').first<{ observed_at: string; fleet_json: string; energy_json: string }>(),
+ ])
+ const rows = byHour(stored.results)
+ const daysState = !days ? 'empty' : nowMs - Date.parse(days.observed_at) > 2 * DAY_MS ? 'stale' : 'visible'
+ const fleet = { state: daysState, observed_at: days?.observed_at ?? null, meaning: 'exact daily reports for closed UTC days, not users or unique boxes; private view only',
+ days: days ? (JSON.parse(days.fleet_json) as FleetDay[]).sort((a, b) => a.date.localeCompare(b.date)) : [] }
+ const energy = { state: daysState, observed_at: days?.observed_at ?? null, meaning: 'exact daily energy reports for closed dates, not homes; private view only',
+ days: days ? (JSON.parse(days.energy_json) as ExactEnergyDay[]).sort((a, b) => a.date.localeCompare(b.date)) : [] }
+ if (!rows.length) return { state: 'empty', hours: [], fleet, energy }
+ const weekRows = rows.filter(r => Date.parse(r.hour) >= nowMs - 7 * DAY_MS)
+ const monthRows = rows.filter(r => r.hour >= monthStart)
+ const totals: Record = {}
+ for (const r of weekRows) for (const [k, n] of Object.entries(r.counters)) totals[k] = (totals[k] ?? 0) + n
+ const latest = rows.at(-1)!
+ const g = (key: string) => latest.gauges[key]
+ const month = monthRows.map(r => r.counters)
+ const bytesOut = sumOf(month, 'bytes_out_box') + sumOf(month, 'bytes_out_app')
+ const byStart = new Map(rows.map(r => [Date.parse(r.hour), r]))
+ const lastHour = Math.floor(nowMs / HOUR_MS) * HOUR_MS - HOUR_MS
+ return {
+ state: nowMs - Date.parse(latest.hour) <= 3 * HOUR_MS ? 'visible' : 'stale',
+ meaning: 'sums over the whole service each hour; no home, device or address',
+ latest_hour: latest.hour,
+ now: {
+ boxes: g('homes_connected') ?? g('sockets_box') ?? null, phones: g('sockets_app') ?? null, homes: g('homes') ?? null,
+ box_quic_percent: share(g('transport_sockets_box_quic')?.mean, g('transport_sockets_box_websocket')?.mean),
+ app_quic_percent: share(g('transport_sockets_app_quic')?.mean, g('transport_sockets_app_websocket')?.mean),
+ },
+ month: { start: monthStart, bytes_out: bytesOut, tun_bytes: sumOf(month, 'tun_bytes_up') + sumOf(month, 'tun_bytes_down'),
+ budget_bytes: EGRESS_BUDGET_BYTES, budget_percent: Math.round(bytesOut / EGRESS_BUDGET_BYTES * 1000) / 10,
+ hours_recorded: monthRows.length, hours_so_far: Math.max(0, Math.floor((nowMs - monthStartMs) / HOUR_MS)),
+ note: 'Message bytes the service wrote to boxes and phones in the hours recorded; TLS and framing come on top. The allowance is AWS\'s free data out across the account.' },
+ // The last 48 hours, one slot each; an hour Cloud did not report is a gap.
+ hours: Array.from({ length: 48 }, (_, i) => lastHour - (47 - i) * HOUR_MS).map(start => {
+ const r = byStart.get(start)
+ if (!r) return { hour: new Date(start).toISOString(), missing: true }
+ return {
+ hour: r.hour, seconds: r.seconds, starts: r.starts,
+ boxes_max: r.gauges['sockets_box']?.max ?? null, phones_max: r.gauges['sockets_app']?.max ?? null,
+ bytes_out: (r.counters['bytes_out_box'] ?? 0) + (r.counters['bytes_out_app'] ?? 0),
+ drops: Object.entries(r.counters).filter(([k]) => k.startsWith('drops_')).reduce((n, [, v]) => n + v, 0),
+ forward_p99_ms: slowest(['forward_seconds_envelope', 'forward_seconds_tun_up', 'forward_seconds_tun_down'].map(k => quantileOf(r.latency[k], 'p99'))),
+ }
+ }),
+ week: {
+ hours_recorded: weekRows.length,
+ drops: prefixed(totals, 'drops_'),
+ push: {
+ alert: pushResults(totals, 'apns_alert_'), widgets: pushResults(totals, 'apns_widgets_'), browser: pushResults(totals, 'webpush_'),
+ breaker_opened: totals['apns_breaker_open'] ?? 0, coalesced: totals['push_coalesced'] ?? 0,
+ },
+ mcp: { outcomes: prefixed(totals, 'mcp_'), p99_ms_max: slowestOf(weekRows, 'mcp_call_seconds') },
+ wrong_shard: prefixed(totals, 'wrong_shard_'),
+ },
+ latency_24h: ['forward_seconds_envelope', 'forward_seconds_tun_up', 'forward_seconds_tun_down', 'handshake_seconds_box', 'handshake_seconds_app', 'handshake_seconds_once', 'mcp_call_seconds']
+ .map(key => {
+ const day = rows.filter(r => Date.parse(r.hour) >= nowMs - DAY_MS)
+ return { name: key, n: day.reduce((t, r) => t + (r.latency[key]?.n ?? 0), 0), p99_ms_max: slowestOf(day, key), p50_ms_max: slowestOf(day, key, 'p50') }
+ }),
+ deploys: rows.filter(r => r.starts > 0 || (r.counters['drains'] ?? 0) > 0).map(r => ({
+ hour: r.hour, starts: r.starts, seconds: r.seconds, drains: r.counters['drains'] ?? 0, drained_sockets: r.counters['drained_sockets'] ?? 0,
+ left_open: r.counters['drain_left_open'] ?? 0, box_handshakes: r.latency['handshake_seconds_box']?.n ?? 0, box_handshake_p99_ms: quantileOf(r.latency['handshake_seconds_box'], 'p99'),
+ })).reverse(),
+ fleet,
+ energy,
+ }
+}
+
+function pushResults(totals: Record, prefix: string) {
+ const r = Object.fromEntries(prefixed(totals, prefix).map(x => [x.name, x.count]))
+ const sent = Object.values(r).reduce((n, c) => n + c, 0)
+ return { results: r, sent, ok_percent: sent > 0 ? Math.round((r['ok'] ?? 0) / sent * 1000) / 10 : null }
+}
diff --git a/stats/src/retention.ts b/stats/src/retention.ts
index ea92544..622e977 100644
--- a/stats/src/retention.ts
+++ b/stats/src/retention.ts
@@ -8,6 +8,7 @@ export const RETENTION_DAYS = {
relaySnapshots: 90,
fleetDaily: 90,
collectorRuns: 90,
+ cloudOps: 90,
} as const
export interface RetentionCutoffs {
@@ -18,6 +19,7 @@ export interface RetentionCutoffs {
relaySnapshots: string
fleetDaily: string
collectorRuns: string
+ cloudOps: string
}
export function retentionCutoffs(nowMs: number): RetentionCutoffs {
@@ -29,6 +31,7 @@ export function retentionCutoffs(nowMs: number): RetentionCutoffs {
relaySnapshots: rollingCutoff(nowMs, RETENTION_DAYS.relaySnapshots),
fleetDaily: utcDayCutoff(nowMs, RETENTION_DAYS.fleetDaily, true),
collectorRuns: rollingCutoff(nowMs, RETENTION_DAYS.collectorRuns),
+ cloudOps: rollingCutoff(nowMs, RETENTION_DAYS.cloudOps),
}
}
@@ -46,6 +49,8 @@ export async function pruneStoredData(db: D1Database, nowMs: number): Promise {
+ await reset()
+ await applyD1Migrations(env.DB, env.TEST_MIGRATIONS)
+})
+afterEach(() => { vi.unstubAllGlobals() })
+
+const hourOf = (ms: number) => new Date(Math.floor(ms / HOUR) * HOUR).toISOString().replace('.000Z', 'Z')
+
+// One settled hour, as ftwcloudd writes it.
+function hour(ms: number, over: Record = {}) {
+ return {
+ hour: hourOf(ms), seconds: 3600, starts: 0,
+ counters: { bytes_out_box: 2_000_000, bytes_out_app: 3_000_000, tun_bytes_up: 400_000, tun_bytes_down: 100_000, drops_tun_rate_up: 2, drops_phone_too_slow: 0,
+ apns_alert_ok: 9, apns_alert_gone: 1, webpush_ok: 2, mcp_ok: 30, mcp_tool_error: 2, mcp_refused: 1, drains: 0, drained_sockets: 0, drain_left_open: 0 },
+ gauges: { sockets_box: { min: 3, mean: 4.5, max: 6 }, sockets_app: { min: 0, mean: 1.25, max: 2 }, homes_connected: { min: 3, mean: 4, max: 5 },
+ transport_sockets_box_quic: { min: 2, mean: 3, max: 4 }, transport_sockets_box_websocket: { min: 1, mean: 1, max: 1 } },
+ latency_ms: { forward_seconds_tun_up: { n: 120, p50: 0.25, p95: 1, p99: 2.5 }, handshake_seconds_box: { n: 4, p50: 5, p95: 25, p99: 25 },
+ mcp_call_seconds: { n: 32, p50: 100, p95: 500, p99: null } },
+ ...over,
+ }
+}
+
+const health = () => ({ plan_solve: { '0-1s': 1, '5s+': 1 }, plan_fallbacks: { '0': 2 }, driver_errors: { '0-1%': 2 }, update: { none: 1, failed: 1 } })
+const daysBack = (nowMs: number, n: number) => new Date(nowMs - n * 24 * HOUR).toISOString().slice(0, 10)
+function feed(nowMs: number, hours = [hour(nowMs - 3 * HOUR), hour(nowMs - 2 * HOUR)], over: Record = {}) {
+ return {
+ schema: 'ftw.cloud-ops/1',
+ meaning: 'sums over the whole service, and closed daily totals over all reports; no home, device or address',
+ shard: 's1', observed_at: new Date(nowMs).toISOString().replace(/\.\d{3}Z$/, 'Z'), hours,
+ fleet_days: [{ date: daysBack(nowMs, 1), reports: 2, ftw_versions: { 'v0.155.1-beta.1': 2 }, channels: { beta: 2 }, drivers: { sungrow: 2, easee_cloud: 1 },
+ battery_kwh: { '5-15': 2 }, price_zones: { SE3: 2 }, install_age: { '1-6m': 2 } }],
+ energy_days: [{ date: daysBack(nowMs, 4), reports: 2, pv_generation_kwh: 15, grid_import_kwh: 40, grid_export_kwh: 4, battery_charge_kwh: 9, battery_discharge_kwh: 8, vehicle_charge_kwh: 7 }],
+ ...over,
+ }
+}
+
+describe('FTW Cloud ops feed', () => {
+ // The fixture is ftwcloudd's own answer (srcfl/ftw-cloud internal/ops),
+ // with test series renamed: two hours, a closed fleet day of two reports,
+ // one of them ftw.fleet/2 with health, and an energy date of two reports.
+ it('takes a feed ftwcloudd wrote', () => {
+ const now = Date.parse(written.observed_at)
+ expect(parseOps(written, now)).toEqual(written)
+ expect(written.hours[0]!.latency_ms['example_slow_seconds']).toEqual({ n: 1, p50: null, p95: null, p99: null })
+ expect(written.fleet_days[0]!.reports).toBe(2)
+ expect((written.fleet_days[0] as Record)['update']).toEqual({ ok: 1 })
+ expect(written.energy_days[0]!.grid_import_kwh).toBe(40)
+ })
+
+ it('takes exactly the fields ftwcloudd writes and refuses anything that could name a home', () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ const good = feed(now)
+ expect(parseOps(good, now)).toEqual(good)
+ const variants: [string, (f: any) => void][] = [
+ ['a home id at the top', f => { f.home = 'canaryhomecanaryhomecanary' }],
+ ['a device on an hour', f => { f.hours[0].device = 'd1' }],
+ ['an address on a gauge', f => { f.hours[0].gauges.sockets_box.addr = '192.0.2.1' }],
+ ['a home on a latency', f => { f.hours[0].latency_ms.handshake_seconds_box.home = 'x' }],
+ ['a box id on a fleet day', f => { f.fleet_days[0].box = 'b' }],
+ ['a series name with a dot', f => { f.hours[0].counters['home.abc'] = 1 }],
+ ['a series name in capitals', f => { f.hours[0].counters['Home'] = 1 }],
+ ['a text counter', f => { f.hours[0].counters.drops_x = '1' }],
+ ['a negative counter', f => { f.hours[0].counters.drops_x = -1 }],
+ ['a fractional counter', f => { f.hours[0].counters.drops_x = 1.5 }],
+ ['a gauge mean above its max', f => { f.hours[0].gauges.sockets_box.mean = 7 }],
+ ['quantiles out of order', f => { f.hours[0].latency_ms.handshake_seconds_box.p95 = 1 }],
+ ['a quantile after a null one', f => { f.hours[0].latency_ms.mcp_call_seconds = { n: 1, p50: null, p95: 1, p99: 1 } }],
+ ['quantiles without a count', f => { f.hours[0].latency_ms.mcp_call_seconds = { n: 0, p50: 1, p95: 1, p99: 1 } }],
+ ['an hour still open', f => { f.hours.push(hour(now)) }],
+ ['hours out of order', f => { f.hours.reverse() }],
+ ['an hour off the clock', f => { f.hours[0].hour = '2026-10-24T03:30:00Z' }],
+ ['an hour past 91 days', f => { f.hours.unshift(hour(now - 92 * 24 * HOUR)) }],
+ ['more than a week of hours', f => { f.hours = Array.from({ length: 169 }, (_, i) => hour(now - (170 - i) * HOUR)) }],
+ ['an old observation', f => { f.observed_at = new Date(now - 3 * HOUR).toISOString() }],
+ ['another schema', f => { f.schema = 'ftw.cloud-ops/2' }],
+ ['another meaning', f => { f.meaning = 'per home' }],
+ ['a fleet label above the reports', f => { f.fleet_days[0].drivers.sungrow = 3 }],
+ ['a fleet label of zero', f => { f.fleet_days[0].drivers.sungrow = 0 }],
+ ['a fleet day in the future', f => { f.fleet_days[0].date = '2026-10-25' }],
+ ['a fleet day still open', f => { f.fleet_days[0].date = '2026-10-24' }],
+ ['an energy date still open', f => { f.energy_days[0].date = '2026-10-21' }],
+ ['a home on an energy day', f => { f.energy_days[0].home = 'x' }],
+ ['an energy sum above a report\'s most', f => { f.energy_days[0].pv_generation_kwh = 2001 }],
+ ['an energy day without reports', f => { f.energy_days[0].reports = 0 }],
+ // The public feed may withhold a sum few reports had (null); the exact
+ // internal feed never does.
+ ['a withheld energy sum', f => { f.energy_days[0].vehicle_charge_kwh = null }],
+ ['an array for counters', f => { f.hours[0].counters = [] }],
+ ['a health field alone', f => { f.fleet_days[0].plan_solve = { '0-1s': 2 } }],
+ ['health with a word off the list', f => { Object.assign(f.fleet_days[0], health(), { update: { 'v0.155.0': 1 } }) }],
+ ['health above the reports', f => { Object.assign(f.fleet_days[0], health(), { driver_errors: { '0': 2, '1%+': 1 } }) }],
+ ['health with a home', f => { Object.assign(f.fleet_days[0], health(), { home: 'x' }) }],
+ ]
+ for (const [what, change] of variants) {
+ const f = structuredClone(good)
+ change(f)
+ expect(() => parseOps(f, now), what).toThrow()
+ }
+ })
+
+ it('takes the health counts of reports that carry them', () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ const f = feed(now)
+ Object.assign(f.fleet_days[0]!, health())
+ expect(parseOps(f, now)).toEqual(f)
+ })
+
+ it('waits without a key and asks with it, from the newest hour kept', async () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ const calls: { url: string; auth: string | null; redirect?: string | undefined }[] = []
+ vi.stubGlobal('fetch', async (input: RequestInfo | URL, init?: RequestInit) => {
+ calls.push({ url: String(input), auth: new Headers(init?.headers).get('authorization'), redirect: init?.redirect })
+ return Response.json(feed(now))
+ })
+ await collectOps(env, now)
+ expect(calls).toHaveLength(0)
+ expect(await env.DB.prepare("SELECT ok, detail FROM collector_runs WHERE source='cloud:ops'").first()).toEqual({ ok: 1, detail: 'awaiting key' })
+
+ const keyed = { ...env, CLOUD_OPS_KEY: key }
+ await collectOps(keyed, now)
+ expect(calls[0]).toEqual({ url: OPS_URL + '?since=' + encodeURIComponent('2026-07-26T06:00:00Z'), auth: 'Bearer ' + key, redirect: 'manual' })
+ const rows = await env.DB.prepare('SELECT shard, hour, seconds, starts FROM cloud_ops_hours ORDER BY hour').all()
+ expect(rows.results).toEqual([
+ { shard: 's1', hour: '2026-10-24T03:00:00.000Z', seconds: 3600, starts: 0 },
+ { shard: 's1', hour: '2026-10-24T04:00:00.000Z', seconds: 3600, starts: 0 },
+ ])
+
+ // The next pull starts an hour before the newest kept, and a restart's
+ // late part replaces that hour.
+ const later = now + HOUR
+ vi.stubGlobal('fetch', async (input: RequestInfo | URL) => {
+ calls.push({ url: String(input), auth: null })
+ return Response.json(feed(later, [hour(now - 2 * HOUR, { seconds: 3600, starts: 1 }), hour(now - HOUR)]))
+ })
+ await collectOps(keyed, later)
+ expect(calls[1]!.url).toBe(OPS_URL + '?since=' + encodeURIComponent('2026-10-24T03:00:00Z'))
+ const after = await env.DB.prepare('SELECT hour, starts FROM cloud_ops_hours ORDER BY hour').all()
+ expect(after.results).toEqual([
+ { hour: '2026-10-24T03:00:00.000Z', starts: 0 },
+ { hour: '2026-10-24T04:00:00.000Z', starts: 1 },
+ { hour: '2026-10-24T05:00:00.000Z', starts: 0 },
+ ])
+ })
+
+ it('treats a Cloud without the key as waiting, and a refusal, a redirect or bad data as a failure', async () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ const keyed = { ...env, CLOUD_OPS_KEY: key }
+ vi.stubGlobal('fetch', async () => new Response(null, { status: 404 }))
+ await collectOps(keyed, now)
+ expect(await env.DB.prepare("SELECT ok, detail FROM collector_runs WHERE source='cloud:ops'").first()).toEqual({ ok: 1, detail: 'awaiting key on Cloud' })
+ for (const answer of [
+ () => new Response(null, { status: 401 }),
+ () => new Response(null, { status: 302, headers: { location: 'https://elsewhere.example/' } }),
+ () => Response.json({ ...feed(now), home: 'canaryhomecanaryhomecanary' }),
+ () => new Response('x'.repeat(1024 * 1024 + 1)),
+ ]) {
+ vi.stubGlobal('fetch', async () => answer())
+ await expect(collectOps(keyed, now)).rejects.toThrow()
+ expect(await env.DB.prepare("SELECT ok FROM collector_runs WHERE source='cloud:ops'").first('ok')).toBe(0)
+ }
+ expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM cloud_ops_hours').first('n')).toBe(0)
+ })
+
+ it('stores only sums and exact day totals, under these columns', async () => {
+ const columns = async (table: string) => (await env.DB.prepare(`PRAGMA table_info(${table})`).all<{ name: string }>()).results.map(c => c.name).sort()
+ expect(await columns('cloud_ops_hours')).toEqual(['counters_json', 'gauges_json', 'hour', 'latency_json', 'observed_at', 'seconds', 'shard', 'starts'])
+ expect(await columns('cloud_ops_days')).toEqual(['energy_json', 'fleet_json', 'id', 'observed_at'])
+ })
+
+ it('shows the service and exact fleet days in the private view only', async () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ vi.stubGlobal('fetch', async () => Response.json(feed(now, [hour(now - 3 * HOUR, { starts: 1, counters: { ...hour(0).counters, drains: 1, drained_sockets: 9, drain_left_open: 1 } }), hour(now - 2 * HOUR)])))
+ await collectOps({ ...env, CLOUD_OPS_KEY: key }, now)
+
+ const pub = await dashboardData(env, false, now)
+ expect(pub).not.toHaveProperty('cloud_ops')
+ expect(JSON.stringify(pub)).not.toContain('easee_cloud')
+ expect(JSON.stringify(pub)).not.toContain('2026-10-20')
+ const ops = (await dashboardData(env, true, now))['cloud_ops'] as Record
+ expect(ops.state).toBe('visible')
+ expect(ops.now.boxes).toEqual({ min: 3, mean: 4, max: 5 })
+ expect(ops.now.box_quic_percent).toBe(75)
+ expect(ops.month.bytes_out).toBe(10_000_000)
+ expect(ops.month.budget_bytes).toBe(100e9)
+ expect(ops.hours).toHaveLength(48)
+ expect(ops.hours.filter((h: any) => !h.missing).map((h: any) => h.hour)).toEqual(['2026-10-24T03:00:00.000Z', '2026-10-24T04:00:00.000Z'])
+ expect(ops.hours.at(-1)).toEqual({ hour: '2026-10-24T05:00:00.000Z', missing: true })
+ expect(ops.month).toMatchObject({ hours_recorded: 2, hours_so_far: 558 })
+ expect(ops.week.drops).toEqual([{ name: 'tun_rate_up', count: 4 }])
+ expect(ops.week.push.alert).toEqual({ results: { ok: 18, gone: 2 }, sent: 20, ok_percent: 90 })
+ expect(ops.week.mcp.outcomes).toEqual([{ name: 'ok', count: 60 }, { name: 'tool_error', count: 4 }, { name: 'refused', count: 2 }])
+ expect(ops.deploys).toEqual([expect.objectContaining({ starts: 1, drains: 1, drained_sockets: 9, left_open: 1, box_handshakes: 4, box_handshake_p99_ms: 25 })])
+ expect(ops.fleet.days).toEqual([expect.objectContaining({ date: '2026-10-23', reports: 2, drivers: { sungrow: 2, easee_cloud: 1 } })])
+ expect(ops.energy.days).toEqual([expect.objectContaining({ date: '2026-10-20', reports: 2, pv_generation_kwh: 15, vehicle_charge_kwh: 7 })])
+
+ const { privateKey, publicKey } = await generateKeyPair('RS256', { extractable: true })
+ const jwk = await exportJWK(publicKey)
+ const token = await new SignJWT({ type: 'app', email: 'operator@ftw.energy' }).setProtectedHeader({ alg: 'RS256', kid: 'k' })
+ .setIssuer('https://test.cloudflareaccess.com').setAudience('test-audience').setIssuedAt().setExpirationTime('5m').sign(privateKey)
+ vi.stubGlobal('fetch', async () => Response.json({ keys: [{ ...jwk, alg: 'RS256', use: 'sig', kid: 'k' }] }))
+ const res = await worker.fetch(new Request('https://stats.ftw.energy/api/admin', { headers: { 'cf-access-jwt-assertion': token } }), env)
+ expect(res.status).toBe(200)
+ expect(await res.json>()).toHaveProperty('cloud_ops.fleet.days')
+
+ const page = await worker.fetch(new Request('https://stats.ftw.energy/admin', { headers: { 'cf-access-jwt-assertion': token } }), env)
+ expect(await page.text()).toContain('Service operations')
+ })
+})
+
+describe('the private view of FTW Cloud', () => {
+ const now = Date.UTC(2026, 9, 24, 6, 23)
+ const keyed = () => ({ ...env, CLOUD_OPS_KEY: key })
+ async function store(hours: Record[], over: Record = {}, shard = 's1') {
+ vi.stubGlobal('fetch', async () => Response.json({ ...feed(now, hours as any, over), shard }))
+ await collectOps(keyed(), now)
+ return (await dashboardData(env, true, now))['cloud_ops'] as Record
+ }
+ const slow = { n: 3, p50: 5, p95: null, p99: null }
+
+ it('reads a quantile past the last bucket as the slowest, not as missing', async () => {
+ const ops = await store([
+ hour(now - 3 * HOUR, { latency_ms: { mcp_call_seconds: slow, forward_seconds_tun_up: slow, handshake_seconds_box: slow } }),
+ hour(now - 2 * HOUR, { starts: 1, latency_ms: { mcp_call_seconds: { n: 1, p50: 50, p95: 50, p99: 50 }, handshake_seconds_box: { n: 2, p50: 1, p95: 5, p99: 5 } } }),
+ ])
+ expect(ops.week.mcp.p99_ms_max).toBe('over_10s')
+ expect(ops.latency_24h.find((l: any) => l.name === 'mcp_call_seconds')).toMatchObject({ n: 4, p99_ms_max: 'over_10s', p50_ms_max: 50 })
+ expect(ops.hours.find((h: any) => h.hour === '2026-10-24T03:00:00.000Z').forward_p99_ms).toBe('over_10s')
+ expect(ops.hours.find((h: any) => h.hour === '2026-10-24T04:00:00.000Z').forward_p99_ms).toBe(null)
+ expect(ops.deploys[0].box_handshake_p99_ms).toBe(5)
+ })
+
+ it('keeps the slowest assistant call to the week the panel names', async () => {
+ const ops = await store([
+ hour(now - 20 * 24 * HOUR, { latency_ms: { mcp_call_seconds: { n: 1, p50: 9000, p95: 9000, p99: 9000 } } }),
+ hour(now - 2 * HOUR, { latency_ms: { mcp_call_seconds: { n: 1, p50: 40, p95: 40, p99: 40 } } }),
+ ])
+ expect(ops.week.mcp.p99_ms_max).toBe(40)
+ })
+
+ it('says stale when the hours or the days are old, never that nothing happened', async () => {
+ const old = await store([hour(now - 9 * 24 * HOUR)])
+ expect(old.state).toBe('stale')
+ expect(old.week.hours_recorded).toBe(0)
+ expect(old.week.drops).toEqual([])
+ await env.DB.prepare("UPDATE cloud_ops_days SET observed_at='2026-10-20T00:00:00Z'").run()
+ const days = (await dashboardData(env, true, now))['cloud_ops'] as Record
+ expect(days.fleet.state).toBe('stale')
+ expect(days.energy.state).toBe('stale')
+ })
+
+ it('adds the shards of an hour together', async () => {
+ await store([hour(now - 2 * HOUR)], {}, 's1')
+ const ops = await store([hour(now - 2 * HOUR, { latency_ms: { handshake_seconds_box: slow } })], {}, 's2')
+ expect(ops.now.boxes).toEqual({ min: 6, mean: 8, max: 10 })
+ expect(ops.week.drops).toEqual([{ name: 'tun_rate_up', count: 4 }])
+ expect(ops.deploys).toEqual([])
+ expect(ops.latency_24h.find((l: any) => l.name === 'handshake_seconds_box')).toMatchObject({ n: 7, p99_ms_max: 'over_10s' })
+ })
+
+ it('leaves the public page as it was: no panel, copy or script of it', async () => {
+ const html = await (await worker.fetch(new Request('https://stats.ftw.energy/'), env)).text()
+ for (const word of ['ops-section', 'Service operations', 'group size of 5 too', 'renderOps', 'listInto', 'Energy reports, exact', 'CLOUD_OPS']) {
+ expect(html, word).not.toContain(word)
+ }
+ const page = dashboardDocument(true).html
+ for (const word of ['ops-section', 'Service operations', 'renderOps(data.cloud_ops)']) expect(page).toContain(word)
+ })
+})
diff --git a/stats/test/worker.test.ts b/stats/test/worker.test.ts
index 7f598fa..4517812 100644
--- a/stats/test/worker.test.ts
+++ b/stats/test/worker.test.ts
@@ -317,6 +317,7 @@ describe('project stats Worker', () => {
relaySnapshots: '2026-05-18T12:00:00.000Z',
fleetDaily: '2026-05-19',
collectorRuns: '2026-05-18T12:00:00.000Z',
+ cloudOps: '2026-05-18T12:00:00.000Z',
})
await env.DB.batch([
@@ -389,6 +390,15 @@ describe('project stats Worker', () => {
)
.bind(`source:${finished}`, finished, finished)
),
+ ...['2026-05-18T11:00:00.000Z', '2026-05-18T12:00:00.000Z'].map((hour) =>
+ env.DB
+ .prepare(
+ `INSERT INTO cloud_ops_hours (shard, hour, seconds, starts, counters_json, gauges_json, latency_json, observed_at)
+ VALUES ('s1', ?, 3600, 0, '{}', '{}', '{}', ?)`
+ )
+ .bind(hour, hour)
+ ),
+ env.DB.prepare(`INSERT INTO cloud_ops_days (id, observed_at, fleet_json, energy_json) VALUES (1, '2026-05-18T11:00:00.000Z', '[]', '[]')`),
])
await pruneStoredData(env.DB, nowMs)
@@ -402,9 +412,11 @@ describe('project stats Worker', () => {
'relay_snapshots',
'fleet_daily',
'collector_runs',
+ 'cloud_ops_hours',
]) {
expect(await env.DB.prepare(`SELECT COUNT(*) AS count FROM ${table}`).first('count')).toBe(1)
}
+ expect(await env.DB.prepare('SELECT COUNT(*) AS count FROM cloud_ops_days').first('count')).toBe(0)
})
it('rejects fields that could turn an aggregate into an identity log', async () => {
@@ -643,7 +655,7 @@ it('keeps six-repo snapshots, daily traffic and public Cloud feeds within separa
expect(requests).toBe(count)
}
const runs=await env.DB.prepare('SELECT source,ok FROM collector_runs').all<{source:string;ok:number}>()
- expect(runs.results).toHaveLength(16)
+ expect(runs.results).toHaveLength(17)
expect(runs.results.every(r=>r.ok===1)).toBe(true)
})