diff --git a/.changeset/stats-daily-energy-reports.md b/.changeset/stats-daily-energy-reports.md new file mode 100644 index 0000000..9fb16e9 --- /dev/null +++ b/.changeset/stats-daily-energy-reports.md @@ -0,0 +1,5 @@ +--- +"ftw-webapp": patch +--- + +The project stats page shows energy from FTW Cloud's anonymous daily reports: the newest public day's report count and its solar, grid, battery and vehicle sums, and the public days of the last 30. A day appears once it closes for reports and holds at least five. These are reports, not homes. diff --git a/stats/README.md b/stats/README.md index 3fe1e4e..3576355 100644 --- a/stats/README.md +++ b/stats/README.md @@ -35,6 +35,17 @@ uses the Cloud view in place of the archive and never adds them together. The se policy below supplies installation, capacity and energy cells. Never merge these two report types or treat check-ins as people. +Energy comes from FTW Cloud's anonymous daily energy reports, at +`/v1/energy/public`. Each box sends one report per closed UTC day: the date +and six whole-kWh sums. It goes through an independent Oblivious HTTP relay, +so Cloud never learns which box sent it. Cloud lists a day only once no +report may still name it, four days after it ends, and only with at least +five reports. The Worker parses that feed strictly, keeps the latest copy and +shows the newest public day and the sum over the public days of the last 30. +These are reports, not homes; never add them to the hourly usage cells. Once +Core names the relay, the fleet report takes it too, as a separate message at +its own time; `/v1/fleet/public` does not change. + The signed legacy ingest route stays for compatibility. It accepts group counters only, never routed frames, room handles, box ids, serials, IP addresses, site names or raw household reports. diff --git a/stats/migrations/0004_cloud_energy.sql b/stats/migrations/0004_cloud_energy.sql new file mode 100644 index 0000000..b54df61 --- /dev/null +++ b/stats/migrations/0004_cloud_energy.sql @@ -0,0 +1,6 @@ +-- Latest public daily energy report totals only; never a single report. +CREATE TABLE cloud_energy_view ( + id INTEGER PRIMARY KEY CHECK(id=1), + observed_at TEXT NOT NULL, + view_json TEXT NOT NULL +); diff --git a/stats/src/dashboard.ts b/stats/src/dashboard.ts index 08db618..b46d9cf 100644 --- a/stats/src/dashboard.ts +++ b/stats/src/dashboard.ts @@ -65,14 +65,24 @@ export function dashboardDocument(privateView: boolean): DashboardDocument {
Vehicle charge / discharge—Recorded hours · kWh

The Cloud usage source is not connected yet. Values appear after privacy review, policy activation and updated owner terms.

-

Installation counts do not count people or all FTW boxes. Configured limits do not prove physical control. Energy sums the recorded hours in a 24-hour window; gaps reduce coverage. Withheld cells show no value.

+

Daily energy reports

+
+
Reports—Newest public day
+
Solar generation—Sum of reports · kWh
+
Grid import / export—Sum of reports · kWh
+
Battery charge / discharge—Sum of reports · kWh
+
Vehicle charge—Sum of reports · kWh
+
Public days—Last 30 days
+
+

Daily energy reports appear once FTW Cloud publishes them.

+

Installation counts do not count people or all FTW boxes. Configured limits do not prove physical control. Hourly energy sums the recorded hours in a 24-hour window; gaps reduce coverage. Withheld cells show no value. A daily report is one box's whole UTC day, sent without any id through an independent relay, so FTW Cloud cannot tell which box sent it; a box with a gap in its day sends none. Reports are not homes.

DATA SOURCES

What this page covers

Six active public repositories. Private repositories and archived experiments are outside these totals.

The separate relay is retired. Anonymous daily reports now come through FTW Cloud. The updated Cloud terms include statistics for all Cloud users. Local FTW users can choose to contribute without remote access. Hourly collection awaits privacy review. App Store installs are not collected. Native app development appears below.

-

Missing usage values stay unknown. Public energy cells need at least 20 contributing installations. Historical relay reports remain in the archive below.

+

Missing usage values stay unknown. Public hourly energy cells need at least 20 contributing installations; a public day of energy reports needs at least 5 reports. Historical relay reports remain in the archive below.

@@ -211,7 +221,9 @@ export function dashboardDocument(privateView: boolean): DashboardDocument { } function renderFreshness(freshness, privateMode) { - const labels = [['github', 'GitHub', 2], ['github_traffic', 'Repo traffic', 30], ['site', 'Web app', 2], ['cloud_fleet', 'Daily reports', 2], ['usage', 'Energy reports', 2]]; + // The daily energy feed gains a day once a day, so it is stale after 48 + // hours (ENERGY_STALE_HOURS); the hourly feeds after 2. + const labels = [['github', 'GitHub', 2], ['github_traffic', 'Repo traffic', 30], ['site', 'Web app', 2], ['cloud_fleet', 'Daily reports', 2], ['usage', 'Hourly energy', 2], ['cloud_energy', 'Daily energy', 48]]; const host = $('freshness'); host.replaceChildren(); @@ -287,6 +299,29 @@ export function dashboardDocument(privateView: boolean): DashboardDocument { if (u.state === 'connected') setText('usage-note', 'Window: ' + u.window_start + ' to ' + u.window_end + '. Each value needs at least 20 contributors; a household above 30% of a sum withholds that cell. ' + (Date.now() - Date.parse(u.observed_at) > 2 * 3600000 ? 'Source is stale; these are the last saved values.' : 'Source: FTW Cloud.')); } + function renderEnergyReports(energy) { + const e = energy || {}; + const kwh = (n) => typeof n === 'number' ? new Intl.NumberFormat(undefined, { maximumFractionDigits: 0 }).format(n) : '—'; + const d = e.latest; + // Stale numbers stay readable but never pass for current ones. + const stale = e.stale ? ' · stale source' : ''; + setText('energy-reports', d ? number(d.reports) : '—'); + setText('energy-day', d ? d.date + ' · UTC day' + stale : 'Newest public day'); + setText('energy-solar', kwh(d && d.pv_generation_kwh)); + setText('energy-grid', kwh(d && d.grid_import_kwh) + ' / ' + kwh(d && d.grid_export_kwh)); + setText('energy-battery', kwh(d && d.battery_charge_kwh) + ' / ' + kwh(d && d.battery_discharge_kwh)); + setText('energy-vehicle', kwh(d && d.vehicle_charge_kwh)); + const w = e.window; + setText('energy-window', w ? number(w.days) : '—'); + const solar = w && (w.pv_generation_days === w.days ? kwh(w.pv_generation_kwh) + ' kWh solar' : kwh(w.pv_generation_kwh) + ' kWh solar on ' + number(w.pv_generation_days) + ' of them'); + setText('energy-window-note', w ? number(w.reports) + ' reports · ' + solar + stale : 'Last 30 days'); + const rule = 'A day appears once it closes for reports, four days after it ends, and needs at least ' + (e.minimum || 5) + ' reports; a sum fewer than ' + (e.minimum || 5) + ' of them had any of shows —. These are reports, not homes.'; + const source = e.stale ? ' Source is stale: last read ' + age(e.observed_at) + '. These are the last saved values.' : ' Source: FTW Cloud.'; + if (e.state === 'pending') setText('energy-note', 'Daily energy reports appear once FTW Cloud publishes them. ' + rule); + if (e.state === 'withheld') setText('energy-note', 'No public day yet. ' + rule + source); + if (e.state === 'connected' && d) setText('energy-note', 'Newest public day ' + d.date + '. ' + rule + source); + } + function renderRepositories(repositories, coverage) { const body = $('repositories'); body.replaceChildren(); @@ -516,6 +551,7 @@ export function dashboardDocument(privateView: boolean): DashboardDocument { renderFreshness(data.freshness || {}, data.mode === 'private'); renderOverview(data); renderUsage(data.usage, data.installation_estimate); + renderEnergyReports(data.energy_reports); renderHeadline(data.github || {}); renderRepositories(data.github && data.github.repositories, data.coverage); renderLine('star-chart', aggregateStateHistory(data.github && data.github.history, 'stars', data.github && data.github.repositories)); @@ -551,6 +587,7 @@ const styles = ` [hidden]{display:none!important} *{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 80% -20%,#252116 0,transparent 34rem),var(--bg);min-height:100vh}a{color:inherit}.topbar{height:58px;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;padding:0 clamp(18px,4vw,56px);position:sticky;top:0;background:color-mix(in srgb,var(--bg) 90%,transparent);backdrop-filter:blur(16px);z-index:5}.brand{display:flex;gap:10px;align-items:center;text-decoration:none;font:700 12px/1 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.18em}.brand-mark{color:var(--accent)}.brand-slash{color:#5e615d}.top-actions{display:flex;align-items:center;gap:16px;font:11px/1 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.08em}.mode{color:var(--accent);border:1px solid #59401c;padding:6px 8px;border-radius:2px}.muted{color:var(--dim)}main{width:min(1280px,calc(100% - 36px));margin:0 auto;padding:64px 0 72px}.hero{display:flex;justify-content:space-between;gap:48px;align-items:flex-end;margin-bottom:54px}.eyebrow{color:var(--accent);font:700 10px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.2em;margin:0 0 12px}h1{font-size:clamp(36px,5vw,72px);line-height:.98;letter-spacing:-.055em;max-width:800px;margin:0}h2{font-size:20px;letter-spacing:-.025em;margin:0}h3{font-size:12px;color:var(--dim);text-transform:uppercase;letter-spacing:.08em;margin:0 0 10px}.lede{color:var(--dim);font-size:17px;max-width:660px;line-height:1.5;margin:20px 0 0}.freshness{min-width:230px;display:grid;gap:8px}.freshness-item{display:grid;grid-template-columns:10px 1fr auto;align-items:center;gap:10px;font-size:12px;color:var(--dim)}.freshness-item strong{font-weight:500;color:var(--fg)}.dot{width:7px;height:7px;border-radius:50%;background:#555;display:inline-block}.dot.ok{background:var(--ok);box-shadow:0 0 12px color-mix(in srgb,var(--ok) 50%,transparent)}.dot.warn{background:var(--warn)}.dot.idle{background:#555}.notice{border:1px solid #674a20;background:#21190e;color:#ffd48f;padding:14px 16px;margin-bottom:24px}.section-head,.panel-head{display:flex;justify-content:space-between;align-items:flex-end;gap:20px}.section-head{margin:0 0 16px}.section-copy{color:var(--dim);font-size:12px;line-height:1.5;max-width:560px;margin:0;text-align:right}.overview-grid,.metric-grid{display:grid;border-top:1px solid var(--line);border-left:1px solid var(--line);margin-bottom:28px}.overview-grid{grid-template-columns:repeat(5,minmax(0,1fr))}.metric-grid{grid-template-columns:repeat(4,minmax(0,1fr));margin-bottom:18px}.metric-card{background:color-mix(in srgb,var(--surface) 92%,transparent);min-height:154px;padding:22px;border-right:1px solid var(--line);border-bottom:1px solid var(--line);display:flex;flex-direction:column}.overview-card{min-height:142px}.metric-card>span,.mini-grid span{font:11px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--dim);text-transform:uppercase;letter-spacing:.08em}.metric-card strong{font-size:42px;line-height:1;margin:auto 0 12px;letter-spacing:-.05em}.metric-card.status-card strong{font-size:30px;letter-spacing:-.035em}.metric-card small{color:var(--dim);font-size:12px;line-height:1.35}.two-col{display:grid;grid-template-columns:1fr 1fr;gap:18px;margin:18px 0}.panel{background:color-mix(in srgb,var(--surface) 94%,transparent);border:1px solid var(--line);padding:22px;margin:18px 0}.two-col>.panel{margin:0}.chart-panel{min-height:330px}.chart{height:235px;margin-top:24px}.chart svg{display:block;width:100%;height:100%;overflow:visible}h2{scroll-margin-top:90px}.mini-grid.usage-grid{grid-template-columns:repeat(3,minmax(0,1fr))}.chart-panel .muted{font-size:11px}.chart-area{fill:color-mix(in srgb,var(--accent) 9%,transparent)}.chart-line{fill:none;stroke:var(--accent);stroke-width:3;stroke-linejoin:round;stroke-linecap:round}.bars{height:230px;display:flex;gap:4px;align-items:flex-end;margin-top:24px;border-bottom:1px solid var(--line);padding:0 2px}.bars.compact{height:150px}.bar-wrap{height:100%;flex:1;min-width:3px;display:flex;flex-direction:column;justify-content:flex-end;align-items:center;gap:8px}.bar{width:100%;max-width:28px;background:linear-gradient(to top,#88520d,var(--accent));border-radius:2px 2px 0 0;min-height:2px}.bar.withheld{background:#4a4c49}.bar.h0{height:2%}.bar.h1{height:5%}.bar.h2{height:10%}.bar.h3{height:15%}.bar.h4{height:20%}.bar.h5{height:25%}.bar.h6{height:30%}.bar.h7{height:35%}.bar.h8{height:40%}.bar.h9{height:45%}.bar.h10{height:50%}.bar.h11{height:55%}.bar.h12{height:60%}.bar.h13{height:65%}.bar.h14{height:70%}.bar.h15{height:75%}.bar.h16{height:80%}.bar.h17{height:85%}.bar.h18{height:90%}.bar.h19{height:95%}.bar.h20{height:100%}.bar-wrap span{font-size:8px;color:var(--dim);height:12px;white-space:nowrap}.table-wrap{overflow-x:auto;margin-top:20px}table{width:100%;border-collapse:collapse;font-size:13px}th{text-align:left;color:var(--dim);font:10px/1 ui-monospace,SFMono-Regular,Menlo,monospace;text-transform:uppercase;letter-spacing:.08em;padding:12px 14px;border-bottom:1px solid var(--line)}td{padding:15px 14px;border-bottom:1px solid #232523;white-space:nowrap}td:first-child{font-weight:650}td a{text-decoration-color:#51544f;text-underline-offset:3px}.repo-role{display:block;color:var(--dim);font-size:11px;font-weight:400;margin-top:4px}.numeric{font-variant-numeric:tabular-nums}.empty{color:var(--dim);padding:30px 4px;text-align:center}.panel-copy,.footnote{color:var(--dim);font-size:13px;line-height:1.5}.big-inline{font-size:28px;font-weight:700;letter-spacing:-.04em}.dimension-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:20px;margin-top:26px}.dimension{border-top:1px solid var(--line);padding-top:14px}.chips{display:flex;gap:7px;flex-wrap:wrap}.chip{font:11px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;color:#d5d3cb;background:var(--raised);border:1px solid var(--line);padding:6px 8px;border-radius:2px}.mini-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:1px;background:var(--line);margin-top:22px}.mini-grid>div{background:var(--raised);padding:18px;display:flex;flex-direction:column;gap:12px}.mini-grid strong{font-size:26px}.ranked-list,.status-list{list-style:none;padding:0;margin:18px 0 0}.ranked-list li{display:flex;justify-content:space-between;gap:18px;padding:12px 0;border-bottom:1px solid var(--line);font-size:13px}.status-list li{display:grid;grid-template-columns:10px 1fr auto;align-items:center;gap:10px;padding:12px 0;border-bottom:1px solid var(--line);font-size:12px}footer{border-top:1px solid var(--line);padding:24px clamp(18px,4vw,56px);display:flex;justify-content:space-between;gap:20px;color:var(--dim);font-size:12px}footer a{color:var(--fg);text-underline-offset:3px} .mini-grid.site-grid{grid-template-columns:repeat(3,1fr)} +.usage-subhead{margin:28px 0 0} @media(max-width:1100px){.overview-grid{grid-template-columns:repeat(3,1fr)}} @media(max-width:900px){.hero{align-items:flex-start;flex-direction:column}.freshness{width:100%;grid-template-columns:repeat(2,1fr)}.section-head{align-items:flex-start;flex-direction:column}.section-copy{text-align:left}.overview-grid,.metric-grid{grid-template-columns:repeat(2,1fr)}.two-col{grid-template-columns:1fr}.dimension-grid{grid-template-columns:repeat(2,1fr)}.mini-grid,.mini-grid.usage-grid{grid-template-columns:repeat(2,1fr)}.mini-grid.site-grid{grid-template-columns:repeat(3,1fr)}} @media(max-width:560px){main{width:min(100% - 24px,1280px);padding-top:38px}.topbar{padding:0 12px}.top-actions #generated{display:none}.hero{margin-bottom:36px;gap:28px}h1{font-size:40px}.freshness{grid-template-columns:1fr}.metric-card{min-height:130px;padding:16px}.metric-card strong{font-size:34px}.panel{padding:16px}.dimension-grid{grid-template-columns:1fr}.mini-grid{grid-template-columns:1fr 1fr}.bars{gap:2px}footer{flex-direction:column;padding:20px 12px}} diff --git a/stats/src/energy.ts b/stats/src/energy.ts new file mode 100644 index 0000000..51c3329 --- /dev/null +++ b/stats/src/energy.ts @@ -0,0 +1,84 @@ +import { saveCollectorRun } from './collector.ts' +import type { AppEnv } from './types.ts' + +// FTW Cloud's anonymous daily energy reports: one report per box and closed +// UTC day, sent through an independent relay, so Cloud never learns which +// box sent it. Cloud lists a date only once no report may still name it. A +// date with fewer than five reports shows null, and so does a sum that +// between one and four of them had any of. These are reports, not homes. +export const energyFields = ['pv_generation_kwh', 'grid_import_kwh', 'grid_export_kwh', 'battery_charge_kwh', 'battery_discharge_kwh', 'vehicle_charge_kwh'] as const +const PUBLIC_ENERGY_MINIMUM = 5 +// A box names one of the last three closed days, so the newest public date is +// four days back. +const OPEN_DAYS = 4 +const MAX_REPORTS = 100_000 +const MAX_KWH = 1000 +export type EnergyDay = { date: string; reports: number | null } & Record +export interface PublicEnergyFeed { schema: 'ftw.energy-public/1'; minimum: number; meaning: string; observed_at: string; days: EnergyDay[] } + +export function parseEnergy(value: unknown, nowMs: number): PublicEnergyFeed { + const v = value as Record + if (!v || typeof v !== 'object' || Array.isArray(v) || Object.keys(v).sort().join() !== ['days','meaning','minimum','observed_at','schema'].join() || v['schema'] !== 'ftw.energy-public/1' || v['minimum'] !== PUBLIC_ENERGY_MINIMUM || v['meaning'] !== 'daily reports, not homes') throw new Error('invalid public energy') + const observed = typeof v['observed_at'] === 'string' ? Date.parse(v['observed_at']) : NaN + if (!Number.isFinite(observed) || Math.abs(nowMs-observed)>2*3600_000 || !Array.isArray(v['days']) || v['days'].length>31) throw new Error('invalid energy window') + // The earlier of the two clocks: a feed stamped ahead of this Worker must + // not make a day still open for reports look closed. + const today = Math.floor(Math.min(observed, nowMs)/86400_000)*86400_000 + const newest = new Date(today-OPEN_DAYS*86400_000).toISOString().slice(0,10), oldest = new Date(today-30*86400_000).toISOString().slice(0,10) + const seen = new Set() + for (const item of v['days']) { + const d = item as Record, date = d && typeof d === 'object' && typeof d['date'] === 'string' ? d['date'] : '' + if (!d || typeof d !== 'object' || Array.isArray(d) || Object.keys(d).sort().join() !== ['date','reports',...energyFields].sort().join() || !/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(Date.parse(date)) || new Date(date).toISOString().slice(0,10)!==date || date>newest || date d[k] !== null)) throw new Error('small energy count') + continue + } + if (!Number.isSafeInteger(reports) || (reports as number)MAX_REPORTS) throw new Error('small energy count') + // A sum that one to four reports had any of is withheld as null. + for (const k of energyFields) if (d[k] !== null && (!Number.isSafeInteger(d[k]) || (d[k] as number)<0 || (d[k] as number)>(reports as number)*MAX_KWH)) throw new Error('invalid energy sum') + } + return v as unknown as PublicEnergyFeed +} + +export async function collectEnergy(env: AppEnv, nowMs = Date.now()) { + const started = new Date(nowMs).toISOString() + try { + const r=await fetch('https://cloud.ftw.energy/v1/energy/public',{redirect:'manual',headers:{accept:'application/json'},signal:AbortSignal.timeout(15000)}) + // A Cloud that does not serve the route yet is a source still to come, + // not a failed collection. + if (r.status===404) {await r.body?.cancel();await saveCollectorRun(env.DB,'cloud:energy',started,true,'awaiting source');return} + if (!r.ok) {await r.body?.cancel();throw new Error('energy unavailable')} + const reader=r.body?.getReader(); if (!reader) throw new Error('empty energy') + const parts:Uint8Array[]=[];let size=0 + try {while(true){const {done,value}=await reader.read();if(done)break;size+=value.length;if(size>64*1024)throw new Error('large energy');parts.push(value)}}finally{await reader.cancel()} + const bytes=new Uint8Array(size);let offset=0;for(const p of parts){bytes.set(p,offset);offset+=p.length} + const feed=parseEnergy(JSON.parse(new TextDecoder().decode(bytes)),nowMs) + await env.DB.prepare('INSERT INTO cloud_energy_view(id,observed_at,view_json) VALUES(1,?,?) ON CONFLICT(id) DO UPDATE SET observed_at=excluded.observed_at,view_json=excluded.view_json').bind(feed.observed_at,JSON.stringify(feed)).run() + await saveCollectorRun(env.DB,'cloud:energy',started,true,'public daily report totals') + } catch(e){await saveCollectorRun(env.DB,'cloud:energy',started,false,'energy source unavailable or invalid');throw e} +} + +// The newest public date with a count, and each sum over the public dates +// that show it. Never a value Cloud did not publish. A daily feed is stale +// after 48 hours, not the 2 of the hourly feeds: Cloud adds a day once a day. +export const ENERGY_STALE_HOURS = 48 +export function energySummary(feed: PublicEnergyFeed | null, nowMs: number) { + const meaning = 'daily reports, not homes' + if (!feed) return { state: 'pending', stale: false, meaning, observed_at: null, latest: null, window: null } + const counted = feed.days.filter(d => d.reports !== null).sort((a,b)=>a.date = { days: counted.length, first: counted.at(-1)?.date ?? null, last: counted[0]?.date ?? null, reports: counted.reduce((n,d)=>n+(d.reports as number),0) } + for (const k of energyFields) { + const shown = counted.filter(d => d[k] !== null) + window[k] = shown.length ? shown.reduce((n,d)=>n+(d[k] as number),0) : null + window[k.replace(/_kwh$/, '_days')] = shown.length + } + return { + state: counted.length ? 'connected' : 'withheld', + stale: nowMs-Date.parse(feed.observed_at) > ENERGY_STALE_HOURS*3600_000, + meaning, minimum: feed.minimum, observed_at: feed.observed_at, + latest: counted[0] ?? null, + window: counted.length ? window : null, + } +} diff --git a/stats/src/index.ts b/stats/src/index.ts index 1712d5b..3405b1d 100644 --- a/stats/src/index.ts +++ b/stats/src/index.ts @@ -1,5 +1,6 @@ import { accessIdentity } from './access.ts' import { dashboardDocument } from './dashboard.ts' +import { collectEnergy } from './energy.ts' import { collectFleet } from './fleet.ts' import { collectUsage } from './usage.ts' import { collectGitHub } from './github.ts' @@ -86,6 +87,7 @@ export default { } try { await collectUsage(env) } catch { failures.push('cloud:usage') } try { await collectFleet(env) } catch { failures.push('cloud:fleet') } + try { await collectEnergy(env) } catch { failures.push('cloud:energy') } } else { throw new Error('unknown collection schedule') } await pruneStoredData(env.DB, controller.scheduledTime) if (failures.length > 0) throw new Error('one or more scheduled collectors failed') diff --git a/stats/src/retention.ts b/stats/src/retention.ts index 29d2542..ea92544 100644 --- a/stats/src/retention.ts +++ b/stats/src/retention.ts @@ -43,6 +43,7 @@ export async function pruneStoredData(db: D1Database, nowMs: number): Promise() const feed = cloudFleet ? JSON.parse(cloudFleet.view_json) as PublicFleetFeed : null + const cloudEnergy = await env.DB.prepare('SELECT observed_at, view_json FROM cloud_energy_view WHERE id=1').first<{ observed_at: string; view_json: string }>() const repoOrder = parseRepos(env.GITHUB_REPOS) const repositories = [...latest.results] .filter((row) => repoOrder.includes(row.repo)) @@ -243,10 +245,11 @@ export async function dashboardData( native_app: { state: 'not_collected', meaning: 'App Store installs and app sessions are not collected' }, fleet: { source: feed ? 'cloud.ftw.energy' : 'relay.ftw.energy', state: feed ? 'connected' : 'retired' }, }, - freshness: { ...freshness, cloud_fleet: cloudFleet?.observed_at ?? null, usage: usage?.observed_at ?? null }, + freshness: { ...freshness, cloud_fleet: cloudFleet?.observed_at ?? null, usage: usage?.observed_at ?? null, cloud_energy: cloudEnergy?.observed_at ?? null }, installation_estimate: installationEstimate(feed?.days ?? (fleetView['days'] as {date:string;reports:number|null}[] ?? []), nowMs, cloudFleet?.observed_at ?? freshness['fleet'] ?? null, feed ? 'cloud.ftw.energy' : 'retired relay archive'), fleet_current: feed, usage: { ...(usage ? JSON.parse(usage.view_json) as Record : { minimum: 20 }), state: usageState }, + energy_reports: energySummary(cloudEnergy ? JSON.parse(cloudEnergy.view_json) as PublicEnergyFeed : null, nowMs), github, site: siteSummary(siteTraffic.results, env.SITE_HOSTNAME, nowMs), fleet: fleetView, diff --git a/stats/test/worker.test.ts b/stats/test/worker.test.ts index 86d8e6f..7f598fa 100644 --- a/stats/test/worker.test.ts +++ b/stats/test/worker.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import worker from '../src/index.ts' import { collectUsage, parseUsage, usageKeys } from '../src/usage.ts' import { installationEstimate, parseFleet, collectFleet } from '../src/fleet.ts' +import { collectEnergy, energyFields, parseEnergy, type EnergyDay, type PublicEnergyFeed } from '../src/energy.ts' import { collectGitHub } from '../src/github.ts' import { ingestRelay } from '../src/relay.ts' import { pruneStoredData, retentionCutoffs, utcDayCutoff } from '../src/retention.ts' @@ -575,6 +576,7 @@ describe('project stats Worker', () => { vi.stubGlobal('fetch', async (input: RequestInfo | URL) => { const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input : input.url) if (url.hostname === 'cloud.ftw.energy' && url.pathname === '/v1/fleet/public') return Response.json(fleetView(Date.now())) + if (url.hostname === 'cloud.ftw.energy' && url.pathname === '/v1/energy/public') return Response.json(energyView(Date.now())) if (url.hostname === 'cloud.ftw.energy') return Response.json(usageView(Date.now())) if (url.hostname === 'api.cloudflare.com') { siteCalls += 1 @@ -625,7 +627,7 @@ it('keeps six-repo snapshots, daily traffic and public Cloud feeds within separa if(url.hostname==='api.cloudflare.com') return Response.json({data:{viewer:{zones:[{traffic:[]}]}},errors:null}) if(url.hostname==='cloud.ftw.energy') return url.pathname==='/v1/stats' ? new Response(null,{status:503,headers:{'FTW-Statistics-State':'disabled'}}) - : Response.json(fleetView(Date.now())) + : url.pathname==='/v1/energy/public' ? Response.json(energyView(Date.now())) : Response.json(fleetView(Date.now())) if(url.pathname==='/search/issues') return Response.json({total_count:0}) if(url.pathname.endsWith('/traffic/views')) return Response.json({views:[]}) if(url.pathname.endsWith('/traffic/clones')) return Response.json({clones:[]}) @@ -635,13 +637,13 @@ it('keeps six-repo snapshots, daily traffic and public Cloud feeds within separa return Response.json([]) }) const all={...env,GITHUB_REPOS:repos.join(',')} - for(const [cron,count] of [['3 * * * *',43],['23 * * * *',9],['17 0 * * *',24]] as const){ + for(const [cron,count] of [['3 * * * *',43],['23 * * * *',10],['17 0 * * *',24]] as const){ requests=0 await worker.scheduled({cron,scheduledTime:Date.now(),noRetry(){}},all) expect(requests).toBe(count) } const runs=await env.DB.prepare('SELECT source,ok FROM collector_runs').all<{source:string;ok:number}>() - expect(runs.results).toHaveLength(15) + expect(runs.results).toHaveLength(16) expect(runs.results.every(r=>r.ok===1)).toBe(true) }) @@ -791,8 +793,94 @@ describe('installation estimates',()=>{ }) +function energyView(now: number): PublicEnergyFeed { + const today=Math.floor(now/86400_000)*86400_000 + const days:EnergyDay[]=Array.from({length:27},(_,i)=>{ + const date=new Date(today-(4+i)*86400_000).toISOString().slice(0,10) + if (i===0) return {date,reports:7,pv_generation_kwh:84,grid_import_kwh:210,grid_export_kwh:30,battery_charge_kwh:63,battery_discharge_kwh:56,vehicle_charge_kwh:20} + if (i===1) return {date,reports:5,pv_generation_kwh:60,grid_import_kwh:150,grid_export_kwh:20,battery_charge_kwh:45,battery_discharge_kwh:40,vehicle_charge_kwh:0} + return {date,reports:null,...Object.fromEntries(energyFields.map(k=>[k,null]))} as EnergyDay + }) + return {schema:'ftw.energy-public/1',meaning:'daily reports, not homes',minimum:5,observed_at:new Date(now).toISOString(),days} +} +describe('anonymous daily energy reports',()=>{ + it('accepts only closed days with five or more reports and nothing else',()=>{ + const now=Date.UTC(2026,9,24,9);const v=energyView(now) + expect(parseEnergy(v,now)).toEqual(v) + const bad=(f:(x:ReturnType)=>void,message:string)=>{const x=energyView(now);f(x);expect(()=>parseEnergy(x,now)).toThrow(message)} + bad(x=>{(x as unknown as Record)['home']='secret'},'invalid public energy') + bad(x=>{x.minimum=3},'invalid public energy') + bad(x=>{x.meaning='homes'},'invalid public energy') + bad(x=>{x.observed_at=new Date(now-3*3600_000).toISOString()},'invalid energy window') + bad(x=>{x.days[0]!.reports=4},'small energy count') + bad(x=>{x.days[2]!.pv_generation_kwh=3},'small energy count') + bad(x=>{x.days[0]!.pv_generation_kwh=7001},'invalid energy sum') + bad(x=>{x.days[0]!.grid_export_kwh=-1},'invalid energy sum') + bad(x=>{x.days[0]!.grid_export_kwh=1.5},'invalid energy sum') + bad(x=>{(x.days[0] as Record)['home']='a'},'invalid energy day') + bad(x=>{x.days[0]!.date=new Date(now-3*86400_000).toISOString().slice(0,10)},'invalid energy day') + bad(x=>{x.days[1]!.date=x.days[0]!.date},'invalid energy day') + }) + it('takes the earlier clock, so a feed stamped ahead cannot close a day early',async()=>{ + const now=Date.UTC(2026,9,24,22,30), ahead=Date.UTC(2026,9,25,0,30) + // Stamped ahead, Cloud lists 2026-10-21, which a box may still name here. + expect(energyView(ahead).days[0]!.date).toBe('2026-10-21') + expect(()=>parseEnergy(energyView(ahead),now)).toThrow('invalid energy day') + expect(parseEnergy(energyView(now),now).days[0]!.date).toBe('2026-10-20') + }) + it('reads dates and the stamp only as strings',()=>{ + const now=Date.UTC(2026,9,24,9) + const stamp=energyView(now); (stamp as unknown as Record)['observed_at']=[stamp.observed_at] + expect(()=>parseEnergy(stamp,now)).toThrow('invalid energy window') + const date=energyView(now); (date.days[0] as Record)['date']=[date.days[0]!.date] + expect(()=>parseEnergy(date,now)).toThrow('invalid energy day') + }) + it('takes a sum Cloud withheld and leaves it out of the window',async()=>{ + const now=Date.UTC(2026,9,24,9);const v=energyView(now) + v.days[0]!.vehicle_charge_kwh=null + expect(parseEnergy(v,now)).toEqual(v) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json(v))) + await collectEnergy(env,now) + const data=await dashboardData(env,false,now) as Record + expect(data['energy_reports'].latest.vehicle_charge_kwh).toBeNull() + expect(data['energy_reports'].window).toMatchObject({days:2,vehicle_charge_kwh:0,vehicle_charge_days:1,pv_generation_kwh:144,pv_generation_days:2}) + }) + it('says a feed is stale after 48 hours, on its own clock',async()=>{ + const now=Date.UTC(2026,9,24,9);const v=energyView(now) + for (const d of v.days){d.reports=null;for(const k of energyFields)d[k]=null} + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json(v))) + await collectEnergy(env,now) + const fresh=await dashboardData(env,false,now+47*3600_000) as Record + expect(fresh['energy_reports']).toMatchObject({state:'withheld',stale:false}) + const later=await dashboardData(env,false,now+49*3600_000) as Record + expect(later['energy_reports']).toMatchObject({state:'withheld',stale:true,observed_at:v.observed_at}) + expect(later['freshness']['cloud_energy']).toBe(v.observed_at) + }) + it('shows the newest public day and the window, labelled as reports',async()=>{ + const now=Date.UTC(2026,9,24,9) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json(energyView(now)))) + await collectEnergy(env,now) + const data=await dashboardData(env,false,now) as Record + expect(data['energy_reports']).toMatchObject({state:'connected',meaning:'daily reports, not homes',minimum:5, + latest:{date:'2026-10-20',reports:7,pv_generation_kwh:84}, + window:{days:2,first:'2026-10-19',last:'2026-10-20',reports:12,pv_generation_kwh:144,vehicle_charge_kwh:20}}) + const page=await worker.fetch(new Request('https://stats.ftw.energy/'),env) + expect(await page.text()).toContain('Daily energy reports') + }) + it('waits for a Cloud that does not serve the route yet, and stores nothing invalid',async()=>{ + const now=Date.UTC(2026,9,24,9) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(new Response('not found',{status:404}))) + await expect(collectEnergy(env,now)).resolves.toBeUndefined() + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json({...energyView(now),home:'secret'}))) + await expect(collectEnergy(env,now)).rejects.toThrow() + expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM cloud_energy_view').first('n')).toBe(0) + const data=await dashboardData(env,false,now) as Record + expect(data['energy_reports']).toMatchObject({state:'pending',latest:null}) + }) +}) + describe('Cloud collector redirects in the Workers runtime', () => { - it.each([['usage', collectUsage], ['fleet', collectFleet]] as const)( + it.each([['usage', collectUsage], ['fleet', collectFleet], ['energy', collectEnergy]] as const)( '%s uses a valid edge Request and rejects a redirect without storing it', async (_name, collect) => { const send = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { @@ -805,6 +893,7 @@ describe('Cloud collector redirects in the Workers runtime', () => { expect(send).toHaveBeenCalledTimes(1) expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM usage_snapshots').first('n')).toBe(0) expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM cloud_fleet_view').first('n')).toBe(0) + expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM cloud_energy_view').first('n')).toBe(0) } ) }) diff --git a/tests/stats-dashboard.test.ts b/tests/stats-dashboard.test.ts index a995b63..843391a 100644 --- a/tests/stats-dashboard.test.ts +++ b/tests/stats-dashboard.test.ts @@ -66,7 +66,9 @@ describe('stats dashboard coverage', () => { const items = [...document.querySelectorAll('#freshness .freshness-item')] expect(items.find((item) => item.textContent!.startsWith('Repo traffic'))!.querySelector('.dot.ok')).not.toBeNull() expect(items.find((item) => item.textContent!.startsWith('GitHub'))!.querySelector('.dot.warn')).not.toBeNull() - expect(items.find((item) => item.textContent!.startsWith('Energy reports'))!.querySelector('.dot.idle')).not.toBeNull() + // The hourly usage feed and the daily energy feed each have their own dot. + expect(items.find((item) => item.textContent!.startsWith('Hourly energy'))!.querySelector('.dot.idle')).not.toBeNull() + expect(items.find((item) => item.textContent!.startsWith('Daily energy'))!.querySelector('.dot.idle')).not.toBeNull() expect(items.some(item => item.textContent!.includes('Legacy'))).toBe(false) })