From 28fddd802242cd901a8773734503d14848f979c3 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 07:56:01 +0200 Subject: [PATCH 1/8] feat(stats): send an anonymous daily energy report through an OHTTP relay Once per closed UTC day, 1 to 12 hours after it closes, the box sends the date and six whole-kWh sums (solar, grid import and export, battery charge and discharge, vehicle charge), sealed with Oblivious HTTP (RFC 9458) to FTW Cloud's pinned gateway key and posted to an independent relay. The relay never sees the report and FTW Cloud never sees the box's address. - go/protocol/ohttp: RFC 9458 on crypto/hpke for one suite, with known-length Binary HTTP; the RFC's Appendix A vectors pass byte for byte. - go/protocol/energywire: the report and its strict decoder. - go/internal/energyreport: the complete-day rule on the energy ledger, the schedule and the sender. The relay URL and the gateway key are compile-time pins, both empty, so this release sends nothing. - The fleet report's answer is the gate, shared in one function; a site a partner controls is left out. Settings shows the report beside the fleet report. - ADR 0008 records what each party sees, the limits and the open ePrivacy question. Co-Authored-By: Claude Opus 5.5 --- .changeset/anonymous-daily-energy.md | 5 + docs/adr/0008-ftw-cloud.md | 55 +++ go/cmd/ftw/energy_report.go | 50 +++ go/cmd/ftw/energy_report_test.go | 51 +++ go/cmd/ftw/fleet_ping.go | 11 +- go/cmd/ftw/main.go | 4 + go/internal/api/api.go | 5 + go/internal/api/api_fleet_ping.go | 13 +- go/internal/api/api_fleet_ping_test.go | 49 +++ go/internal/energyreport/day.go | 115 ++++++ go/internal/energyreport/day_test.go | 175 +++++++++ go/internal/energyreport/energyreport.go | 341 ++++++++++++++++++ go/internal/energyreport/energyreport_test.go | 319 ++++++++++++++++ go/internal/energyreport/random.go | 16 + go/protocol/energywire/energy.go | 119 ++++++ go/protocol/energywire/energy_test.go | 92 +++++ go/protocol/ohttp/bhttp.go | 200 ++++++++++ go/protocol/ohttp/bhttp_test.go | 63 ++++ go/protocol/ohttp/ohttp.go | 335 +++++++++++++++++ go/protocol/ohttp/ohttp_test.go | 247 +++++++++++++ web/fleet-ping-tab.test.mjs | 47 +++ web/settings/tabs/fleet.js | 87 ++++- web/style.css | 9 + 23 files changed, 2403 insertions(+), 5 deletions(-) create mode 100644 .changeset/anonymous-daily-energy.md create mode 100644 go/cmd/ftw/energy_report.go create mode 100644 go/cmd/ftw/energy_report_test.go create mode 100644 go/internal/energyreport/day.go create mode 100644 go/internal/energyreport/day_test.go create mode 100644 go/internal/energyreport/energyreport.go create mode 100644 go/internal/energyreport/energyreport_test.go create mode 100644 go/internal/energyreport/random.go create mode 100644 go/protocol/energywire/energy.go create mode 100644 go/protocol/energywire/energy_test.go create mode 100644 go/protocol/ohttp/bhttp.go create mode 100644 go/protocol/ohttp/bhttp_test.go create mode 100644 go/protocol/ohttp/ohttp.go create mode 100644 go/protocol/ohttp/ohttp_test.go diff --git a/.changeset/anonymous-daily-energy.md b/.changeset/anonymous-daily-energy.md new file mode 100644 index 000000000..e8ecde5f7 --- /dev/null +++ b/.changeset/anonymous-daily-energy.md @@ -0,0 +1,5 @@ +--- +"ftw": minor +--- + +The box can send one anonymous energy report a day: the date and that UTC day's solar generation, grid import and export, battery charge and discharge and vehicle charge, in whole kWh. It goes through an independent Oblivious HTTP relay to FTW Cloud's gateway, so the relay never sees the report and FTW Cloud never sees the box's address. It follows the owner's existing statistics answer, skips any day with a gap in the energy record and any site a partner controls, and adds no question or setting. Settings shows the report beside the fleet report. This release names no relay yet, so it sends nothing until a later release pins one. diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index 6915dbf55..15203c3b9 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -350,3 +350,58 @@ even if an owner previously chose to contribute. The owner policy response reports this readiness so native clients can explain the wait. Pausing the service policy stops collection while withdrawal and retention still run. Deploying this code does not sign the DPIA or accept a household's consent. + +### Anonymous daily energy, October 2026 + +Fredrik chose this on 24 October 2026 so that stats.ftw.energy can show +energy without C8's household rows. One new outside part: an independent +Oblivious HTTP relay (RFC 9458). FTW Cloud's `ftwcloudd` is the gateway, as +one route. + +- **Sent.** Once per closed UTC day, 1 to 12 hours after it closes, at a time + drawn afresh each day: the schema, the date, and six whole-kWh sums (solar + generation, grid import and export, battery charge and discharge, vehicle + charge). Nothing else: no id, version, driver, zone or size. A day with a + gap in the energy ledger is skipped, never sent in part + (`go/internal/energyreport`, `go/protocol/energywire`). +- **Who may send.** The fleet report's answer: its switch and the current + project statistics choice. Not a site a partner controls. No new question + or setting. Settings shows the report beside the fleet report. +- **Path.** The box seals the report to FTW Cloud's gateway key with HPKE + (`go/protocol/ohttp`, the standard library's `crypto/hpke`) and posts it to + the relay, which forwards it to `POST /v1/ohttp`. The inner request is + padded to one length. The key and the relay are compile-time pins + (`energyreport.GatewayKey`, `energyreport.RelayURL`). The box never asks + the service for a key, so every box on a release uses the same one and a + key swapped in at the service opens nothing. Both pins are empty until the + relay exists; until then the box sends nothing. +- **Who sees what.** The relay sees the box's address, the time and a + ciphertext of fixed length, never the report. Cloudflare's relay, if + chosen, keeps those logs for about 124 days. The gateway sees the report, + the relay's address and the time, never the box's address. +- **Kept.** Per date: the report count and the six sums, for 30 days. No row, + logged body or address per report. +- **Published.** `GET /v1/energy/public` lists closed dates only, once no + report may still name them, so two readings never differ by one report. A + date with fewer than 5 reports shows null, as `/v1/fleet/public` does. +- **Not linkable.** The energy report shares no field with the fleet report, + and the two leave at times drawn apart. Neither the relay nor the gateway + holds both a box's address and its numbers. + +Limits, plainly: + +- The relay and Sourceful together could join reports to addresses by time. + The relay's independence is the control. +- FTW Cloud holds readings from homes whose owners turned on the readable + copy, including 48 hours of history tiles. The same daily sums could be + computed from them and matched with a report. The gateway does not do this + and keeps nothing to match with, but that is a rule in code, not one the + cryptography enforces. +- An anonymous report cannot prove that a real box sent it. Bounds per field + and a cap per date limit junk to noise. Privacy Pass-style tokens are the + later fix; they are not built. + +Open for Fredrik, not decided here: whether reports that are anonymous to +Sourceful still need the owner's yes under the ePrivacy rule on reading +terminal equipment (Swedish LEK 9:28). The design keeps the existing yes, so +it is covered either way. C8 stays off. diff --git a/go/cmd/ftw/energy_report.go b/go/cmd/ftw/energy_report.go new file mode 100644 index 000000000..f6e4355e9 --- /dev/null +++ b/go/cmd/ftw/energy_report.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "log/slog" + "sync" + + "github.com/srcfl/ftw/go/internal/cloudlink" + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/energyreport" + "github.com/srcfl/ftw/go/internal/state" +) + +// The anonymous daily energy report, wired to the box. What it sends and how +// is decided in go/internal/energyreport; this file only gives it the ledger +// and the owner's answer. + +// startEnergyReport builds the sender and starts its schedule. It is built +// even when nothing may be sent, so Settings can show the report this box +// would send. A build with no relay or gateway key sends nothing at all. +func startEnergyReport(ctx context.Context, cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) *energyreport.Sender { + if st == nil { + return nil + } + sender, err := energyreport.New(energyreport.Options{ + Enabled: energyReportAllowed(cfg, cfgMu, st), + Ledger: st, + KV: st, + ClockSynced: cloudlink.ClockSynced, + }) + if err != nil { + slog.Warn("energy report unavailable", "err", err) + return nil + } + go sender.Run(ctx) + return sender +} + +// energyReportAllowed is the fleet report's answer, and no site a partner +// controls: as for the hourly project statistics, a partner's contract has to +// let such a home in first. +func energyReportAllowed(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) func() bool { + allowed := projectReportsAllowed(cfg, cfgMu, st) + return func() bool { + cfgMu.RLock() + partner := cfg.Nova != nil && cfg.Nova.Enabled + cfgMu.RUnlock() + return !partner && allowed() + } +} diff --git a/go/cmd/ftw/energy_report_test.go b/go/cmd/ftw/energy_report_test.go new file mode 100644 index 000000000..155f83f71 --- /dev/null +++ b/go/cmd/ftw/energy_report_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "sync" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// The energy report has no question of its own. It sends only when the fleet +// report would, and never for a site a partner controls. +func TestTheEnergyReportTakesTheFleetReportsAnswer(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + cfg := &config.Config{} + cfgMu := &sync.RWMutex{} + energy, fleet := energyReportAllowed(cfg, cfgMu, st), projectReportsAllowed(cfg, cfgMu, st) + check := func(name string, wantEnergy, wantFleet bool) { + t.Helper() + if energy() != wantEnergy || fleet() != wantFleet { + t.Fatalf("%s: energy %v fleet %v, want %v %v", name, energy(), fleet(), wantEnergy, wantFleet) + } + } + + check("no answer yet", false, false) + grant, _ := json.Marshal(map[string]any{"statsPolicy": cloudwire.StatsPolicy, "statsId": make([]byte, 16)}) + if err := st.SaveConfig("cloud/grant_v1", string(grant)); err != nil { + t.Fatal(err) + } + check("the owner said yes", true, true) + + cfg.FleetPing = &config.FleetPing{Enabled: false} + check("the daily report switched off", false, false) + cfg.FleetPing.Enabled = true + + cfg.Nova = &config.Nova{Enabled: true} + check("a partner controls the site", false, true) + cfg.Nova = nil + + if err := st.SaveConfig("cloud/disabled_v1", "true"); err != nil { + t.Fatal(err) + } + check("Cloud ended", false, false) +} diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go index 592ebed7e..9d75d47ec 100644 --- a/go/cmd/ftw/fleet_ping.go +++ b/go/cmd/ftw/fleet_ping.go @@ -77,7 +77,7 @@ func startFleetPing( installedAt := firstSeen(st, time.Now()) pinger, err := fleetping.New(fleetping.Options{ - Enabled: func() bool { return fleetPingEnabled(cfg, cfgMu)() && cloudlink.StatisticsChoice(st) }, + Enabled: projectReportsAllowed(cfg, cfgMu, st), Facts: func() fleetping.Facts { // Drivers can be installed without restarting FTW. Read the signed // catalogue at the same time as the config so tomorrow's report and @@ -97,6 +97,15 @@ func startFleetPing( return pinger, nil } +// projectReportsAllowed is the owner's answer for both anonymous daily +// reports, the fleet report and the energy report: the saved switch and the +// project statistics choice. One function, so the two reports cannot come to +// read different answers. +func projectReportsAllowed(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) func() bool { + enabled := fleetPingEnabled(cfg, cfgMu) + return func() bool { return enabled() && cloudlink.StatisticsChoice(st) } +} + // fleetPingEnabled reads the switch, under the config lock, at each send. // // On() and not a second reading of nil. A config posted to /api/config never diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index fb707f741..b3738e412 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2498,6 +2498,9 @@ func main() { if fleetPingErr != nil { slog.Warn("fleet ping unavailable", "err", fleetPingErr) } + // The anonymous daily energy report, under the same answer, through an + // independent relay so that FTW Cloud never learns which box sent it. + energyReport := startEnergyReport(ctx, cfg, cfgMu, st) mutationPolicy := apiMutationPolicy() mutationPolicy.LANAuthEnabled = func() bool { @@ -2520,6 +2523,7 @@ func main() { AppEnroll: appEnrollForAPI(appEnroll, appUplink, st, appLinkEnabled, agents), Agents: agentsForAPI(agents), FleetPing: fleetPinger, + EnergyReport: energyReport, Cloud: cloudForAPI(cloudLink), CfgMu: cfgMu, Cfg: cfg, ConfigPath: *configPath, ConfigApplier: applyConfigChange, diff --git a/go/internal/api/api.go b/go/internal/api/api.go index d9e2a9b4b..166f34e47 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -43,6 +43,7 @@ import ( "github.com/srcfl/ftw/go/internal/drivers" "github.com/srcfl/ftw/go/internal/evcloud" "github.com/srcfl/ftw/go/internal/events" + "github.com/srcfl/ftw/go/internal/energyreport" "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/forecast" "github.com/srcfl/ftw/go/internal/ha" @@ -115,6 +116,10 @@ type Deps struct { // embeddings; GET /api/fleet-ping then says so. FleetPing *fleetping.Pinger + // EnergyReport is the anonymous daily energy report, held for the same + // reason: Settings shows the report it would send. Nil leaves it out. + EnergyReport *energyreport.Sender + // Cloud is the box's FTW Cloud grant and uplink. Nil in tests and // minimal embeddings; the /api/cloud routes then say it is unavailable. Cloud CloudGrant diff --git a/go/internal/api/api_fleet_ping.go b/go/internal/api/api_fleet_ping.go index cc203a6d2..11d317848 100644 --- a/go/internal/api/api_fleet_ping.go +++ b/go/internal/api/api_fleet_ping.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/energyreport" ) // What this box would add to FTW's daily fleet totals. @@ -21,6 +22,9 @@ type fleetPingView struct { FTWRelay bool `json:"ftw_relay"` // Payload is the whole message, exactly as it would be posted. Payload any `json:"payload"` + // Energy is the daily energy report under the same switch, built by the + // call its sender uses. Absent on a box without one. + Energy *energyreport.Preview `json:"energy,omitempty"` } func (s *Server) handleFleetPing(w http.ResponseWriter, r *http.Request) { @@ -39,10 +43,15 @@ func (s *Server) handleFleetPing(w http.ResponseWriter, r *http.Request) { s.deps.CfgMu.RUnlock() endpoint := s.deps.FleetPing.Endpoint() - writeJSON(w, http.StatusOK, fleetPingView{ + view := fleetPingView{ Enabled: enabled, Endpoint: endpoint, FTWRelay: endpoint == config.DefaultFleetPingEndpoint, Payload: s.deps.FleetPing.Payload(), - }) + } + if s.deps.EnergyReport != nil { + p := s.deps.EnergyReport.Preview(r.Context()) + view.Energy = &p + } + writeJSON(w, http.StatusOK, view) } diff --git a/go/internal/api/api_fleet_ping_test.go b/go/internal/api/api_fleet_ping_test.go index 306137d5b..3a30a8826 100644 --- a/go/internal/api/api_fleet_ping_test.go +++ b/go/internal/api/api_fleet_ping_test.go @@ -10,7 +10,9 @@ import ( "time" "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/energyreport" "github.com/srcfl/ftw/go/internal/fleetping" + "github.com/srcfl/ftw/go/internal/state" ) type silentProvider string @@ -149,3 +151,50 @@ func TestFleetPingSaysSoWhenItIsNotRunning(t *testing.T) { t.Fatalf("status %d, want 503", w.Code) } } + +// dayLedger is one site meter that recorded every bucket of every day. +type dayLedger struct{} + +func (dayLedger) EnergyAssets() ([]state.EnergyAsset, error) { + return []state.EnergyAsset{{AssetID: "m/grid_meter", Kind: state.AssetGridMeter, FirstSeenMS: 0, LastSeenMS: 1 << 62}}, nil +} + +func (dayLedger) LoadEnergyHistoryContext(_ context.Context, q state.EnergyHistoryQuery) ([]state.EnergyLedgerPoint, bool, error) { + var out []state.EnergyLedgerPoint + for at := q.SinceMS; at < q.UntilMS; at += state.EnergyLedgerBucketMS { + out = append(out, state.EnergyLedgerPoint{Flow: state.FlowGridImport, BucketStartMS: at, BucketLenMS: state.EnergyLedgerBucketMS, EnergyWh: 50, Quality: "measured"}) + } + return out, false, nil +} + +// The daily energy report rides under the same switch, so Settings shows it +// beside the fleet report, built by the call its sender uses. A build with no +// relay says it sends nothing. +func TestFleetPingShowsTheEnergyReportItWouldSend(t *testing.T) { + cfg := &config.Config{FleetPing: &config.FleetPing{Enabled: true}} + srv := fleetPingServer(t, cfg) + sender, err := energyreport.New(energyreport.Options{Ledger: dayLedger{}, Enabled: func() bool { return true }, + Now: func() time.Time { return time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC) }}) + if err != nil { + t.Fatal(err) + } + srv.deps.EnergyReport = sender + + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/fleet-ping", nil)) + var got struct { + Energy struct { + Relay string `json:"relay"` + Sending bool `json:"sending"` + Day string `json:"day"` + Payload map[string]any `json:"payload"` + } `json:"energy"` + } + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + e := got.Energy + if e.Relay != "" || e.Sending || e.Day != "2026-10-23" || e.Payload["grid_import_kwh"] != 14.0 || e.Payload["date"] != "2026-10-23" || len(e.Payload) != 8 { + t.Fatalf("energy = %+v", e) + } +} diff --git a/go/internal/energyreport/day.go b/go/internal/energyreport/day.go new file mode 100644 index 000000000..dc47bb5de --- /dev/null +++ b/go/internal/energyreport/day.go @@ -0,0 +1,115 @@ +package energyreport + +import ( + "context" + "math" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/protocol/energywire" +) + +// Ledger is the part of the store a day is read from. +type Ledger interface { + EnergyAssets() ([]state.EnergyAsset, error) + LoadEnergyHistoryContext(context.Context, state.EnergyHistoryQuery) ([]state.EnergyLedgerPoint, bool, error) +} + +// usable are the ledger qualities that count as a recorded bucket. A gap, +// reset, invalid read or anything this list does not name makes the day +// incomplete. +var usable = map[string]bool{"measured": true, "recovered": true, "integrated": true, "estimated": true} + +// sums maps a ledger flow to its place in energywire.Fields. +var sums = map[state.EnergyFlow]int{ + state.FlowPVGeneration: 0, + state.FlowGridImport: 1, + state.FlowGridExport: 2, + state.FlowBatteryCharge: 3, + state.FlowBatteryDischarge: 4, + state.FlowVehicleCharge: 5, +} + +const dayMS = int64(24 * time.Hour / time.Millisecond) + +// Day reads the closed UTC day that starts at start. ok is false unless the +// whole day is there. +// +// Whole means: a site meter, and every grid meter, solar, battery and charger +// asset that was in use that day recorded every bucket of it, with nothing the +// ledger marks as a gap, reset or invalid read. An asset that arrived or went +// silent during the day makes it incomplete; one that left before the day +// began is not part of it. A box that misses part of a day sends nothing for +// it rather than a sum that looks like a quiet day. +func Day(ctx context.Context, l Ledger, start time.Time) (energywire.Report, bool, error) { + start = start.UTC().Truncate(24 * time.Hour) + from := start.UnixMilli() + to := from + dayMS + assets, err := l.EnergyAssets() + if err != nil { + return energywire.Report{}, false, err + } + var wh [6]float64 + meter := false + for _, a := range assets { + switch a.Kind { + case state.AssetGridMeter, state.AssetPV, state.AssetBattery, state.AssetVehicleCharger: + default: + continue + } + if a.LastSeenMS < from { + continue + } + if a.FirstSeenMS > from || a.LastSeenMS < to { + return energywire.Report{}, false, nil + } + points, truncated, err := l.LoadEnergyHistoryContext(ctx, state.EnergyHistoryQuery{ + AssetID: a.AssetID, SinceMS: from, UntilMS: to, + BucketMS: state.EnergyLedgerBucketMS, Limit: int(dayMS / state.EnergyLedgerBucketMS), + }) + if err != nil { + return energywire.Report{}, false, err + } + if truncated || !covers(points, from, to) { + return energywire.Report{}, false, nil + } + for _, p := range points { + if i, ok := sums[p.Flow]; ok { + wh[i] += p.EnergyWh + } + } + meter = meter || a.Kind == state.AssetGridMeter + } + if !meter { + return energywire.Report{}, false, nil + } + r := energywire.Report{Schema: energywire.Schema, Date: start.Format(time.DateOnly)} + fields := []*int64{&r.PVGeneration, &r.GridImport, &r.GridExport, &r.BatteryCharge, &r.BatteryDischarge, &r.VehicleCharge} + for i, v := range wh { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 { + return energywire.Report{}, false, nil + } + *fields[i] = int64(math.Round(v / 1000)) + } + return r, r.Valid(), nil +} + +// covers reports whether the points fill [from, to) with usable buckets of +// one width and hold nothing else. +func covers(points []state.EnergyLedgerPoint, from, to int64) bool { + if len(points) == 0 { + return false + } + width := points[0].BucketLenMS + if width <= 0 || (to-from)%width != 0 { + return false + } + seen := map[int64]bool{} + for _, p := range points { + if p.BucketLenMS != width || p.BucketStartMS < from || p.BucketStartMS >= to || (p.BucketStartMS-from)%width != 0 || !usable[p.Quality] { + return false + } + seen[p.BucketStartMS] = true + } + return int64(len(seen)) == (to-from)/width +} diff --git a/go/internal/energyreport/day_test.go b/go/internal/energyreport/day_test.go new file mode 100644 index 000000000..103fccf5f --- /dev/null +++ b/go/internal/energyreport/day_test.go @@ -0,0 +1,175 @@ +package energyreport + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +var day = time.Date(2026, 10, 23, 0, 0, 0, 0, time.UTC) + +const tick = 2 * time.Minute + +type asset struct { + kind state.EnergyAssetKind + flow state.EnergyFlow + watts float64 + counter bool +} + +// The test site: a metered grid, solar, a battery and a charger, read every +// two minutes, the grid meter by its counters and the rest by power. +var site = map[string][]asset{ + "meter:1/grid_meter": {{state.AssetGridMeter, state.FlowGridImport, 1000, true}, {state.AssetGridMeter, state.FlowGridExport, 0, true}}, + "inverter:1/pv": {{state.AssetPV, state.FlowPVGeneration, 500, false}}, + "battery:1/battery": {{state.AssetBattery, state.FlowBatteryCharge, 250, false}, {state.AssetBattery, state.FlowBatteryDischarge, 0, false}}, + "charger:1/vehicle_charger": {{state.AssetVehicleCharger, state.FlowVehicleCharge, 1000, false}}, + "site/observed-consumer": {{state.AssetObservedConsumer, state.FlowConsumerUse, 700, false}}, +} + +func openStore(t *testing.T) *state.Store { + t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// record writes the site from `from` to `to`, in time order and in one +// transaction, leaving out any asset whose id skip returns true for at that +// time. A counter keeps counting while it is not read, as a meter's does. +func record(t *testing.T, st *state.Store, from, to time.Time, skip func(id string, at time.Time) bool) { + t.Helper() + var obs []state.EnergyObservation + for at := from; !at.After(to); at = at.Add(tick) { + for id, flows := range site { + if skip != nil && skip(id, at) { + continue + } + for _, a := range flows { + w, c := a.watts, a.watts*at.Sub(from).Hours() + o := state.EnergyObservation{AssetID: id, DeviceID: id, AssetKind: a.kind, Label: "x", Flow: a.flow, AtMs: at.UnixMilli(), PowerW: &w} + if a.counter { + o.CounterWh, o.PowerW = &c, nil + } + obs = append(obs, o) + } + } + } + if err := st.RecordTickWithOptionalHistory(nil, nil, obs); err != nil { + t.Fatal(err) + } +} + +// sharedDay is the whole test day, written once and only read. +var sharedDay struct { + once sync.Once + dir string + st *state.Store + err error +} + +func TestMain(m *testing.M) { + code := m.Run() + if sharedDay.st != nil { + _ = sharedDay.st.Close() + } + if sharedDay.dir != "" { + _ = os.RemoveAll(sharedDay.dir) + } + os.Exit(code) +} + +func whole(t *testing.T) *state.Store { + t.Helper() + sharedDay.once.Do(func() { + sharedDay.dir, sharedDay.err = os.MkdirTemp("", "energyreport") + if sharedDay.err != nil { + return + } + sharedDay.st, sharedDay.err = state.Open(filepath.Join(sharedDay.dir, "state.db")) + if sharedDay.err == nil { + record(t, sharedDay.st, day.Add(-10*time.Minute), day.Add(24*time.Hour+10*time.Minute), nil) + } + }) + if sharedDay.err != nil { + t.Fatal(sharedDay.err) + } + return sharedDay.st +} + +func wholeDay(t *testing.T, skip func(string, time.Time) bool) *state.Store { + st := openStore(t) + record(t, st, day.Add(-10*time.Minute), day.Add(24*time.Hour+10*time.Minute), skip) + return st +} + +func TestAWholeDayIsSixWholeKWhSums(t *testing.T) { + st := whole(t) + r, ok, err := Day(context.Background(), st, day.Add(5*time.Hour)) + if err != nil || !ok { + t.Fatalf("ok %v err %v", ok, err) + } + if r.Date != "2026-10-23" || r.GridImport != 24 || r.GridExport != 0 || r.PVGeneration != 12 || + r.BatteryCharge != 6 || r.BatteryDischarge != 0 || r.VehicleCharge != 24 || !r.Valid() { + t.Fatalf("%+v", r) + } +} + +func TestAPartDayIsNotSent(t *testing.T) { + for name, skip := range map[string]func(string, time.Time) bool{ + "solar silent for an hour": func(id string, at time.Time) bool { + return id == "inverter:1/pv" && at.After(day.Add(13*time.Hour)) && at.Before(day.Add(14*time.Hour)) + }, + "a battery that stopped at noon": func(id string, at time.Time) bool { + return id == "battery:1/battery" && at.After(day.Add(12*time.Hour)) + }, + "a charger added at noon": func(id string, at time.Time) bool { + return id == "charger:1/vehicle_charger" && at.Before(day.Add(12*time.Hour)) + }, + "no site meter": func(id string, _ time.Time) bool { return id == "meter:1/grid_meter" }, + } { + t.Run(name, func(t *testing.T) { + if r, ok, err := Day(context.Background(), wholeDay(t, skip), day); err != nil || ok { + t.Fatalf("sent %+v (err %v)", r, err) + } + }) + } + // The day after the ledger ends has not happened yet. + if _, ok, _ := Day(context.Background(), whole(t), day.AddDate(0, 0, 1)); ok { + t.Fatal("an unrecorded day was sent") + } +} + +// A meter read by its counters keeps a whole day across an outage: the +// counter carries the three hours' energy over the gap, and the ledger +// spreads it. +func TestACounterCarriesTheMeterAcrossAnOutage(t *testing.T) { + only := map[string]bool{"meter:1/grid_meter": true} + st := wholeDay(t, func(id string, at time.Time) bool { + return !only[id] || (at.After(day.Add(3*time.Hour)) && at.Before(day.Add(6*time.Hour))) + }) + r, ok, err := Day(context.Background(), st, day) + if err != nil || !ok || r.GridImport != 24 || r.PVGeneration != 0 { + t.Fatalf("%+v ok %v err %v", r, ok, err) + } +} + +// An asset that left before the day began is not part of it. +func TestAnAssetRetiredBeforeTheDayIsLeftOut(t *testing.T) { + st := openStore(t) + record(t, st, day.Add(-48*time.Hour), day.Add(24*time.Hour+10*time.Minute), func(id string, at time.Time) bool { + return id == "battery:1/battery" && !at.Before(day.Add(-24*time.Hour)) + }) + r, ok, err := Day(context.Background(), st, day) + if err != nil || !ok || r.BatteryCharge != 0 || r.GridImport != 24 { + t.Fatalf("%+v ok %v err %v", r, ok, err) + } +} diff --git a/go/internal/energyreport/energyreport.go b/go/internal/energyreport/energyreport.go new file mode 100644 index 000000000..100708ac2 --- /dev/null +++ b/go/internal/energyreport/energyreport.go @@ -0,0 +1,341 @@ +// Package energyreport sends one anonymous energy report a day, through an +// independent Oblivious HTTP relay, so that neither the relay nor FTW Cloud +// can tell which home sent it. +// +// The report is six whole-kWh sums of one closed UTC day and its date +// (package energywire). The box seals it to FTW Cloud's gateway key, pinned +// in this release, and posts it to the relay: +// +// - the relay sees the box's address, the time and a ciphertext of fixed +// length, never the report; +// - the gateway sees the report and the relay's address, never the box's. +// It adds the sums to that day's totals and keeps nothing of the report. +// +// Two things keep the report from being joined to the daily fleet report, +// which goes straight to FTW Cloud from the box's own address: it carries no +// field the fleet report carries, and it is sent at its own time, drawn +// afresh for each day from 1 to 12 hours after the day closes. +// +// It runs only when the owner's answer to the project statistics question +// allows the fleet report, and never for a site a partner controls. It adds +// no question or setting of its own. +package energyreport + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "net/http" + "net/url" + "time" + + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// Compile-time settings, as in cloudlink: a relay a household could point +// elsewhere is a relay a support call could point at an attacker's. A local +// test build may set them with -ldflags -X. +var ( + // RelayURL is the OHTTP relay the report goes through, run by someone + // other than Sourceful. Empty until that relay exists: the box then + // sends nothing. + RelayURL = "" + // GatewayKey is FTW Cloud's gateway key configuration (RFC 9458, + // section 3), base64url, as + // https://cloud.ftw.energy/.well-known/ohttp-gateway serves it. The box + // encrypts only to this key and never asks the service for one, so every + // box on a release uses the same key and a key swapped in at the service + // reads nothing. Empty means no gateway key exists for this build. + GatewayKey = "" +) + +// Pinned returns this build's relay and gateway key. ok is false unless both +// are set and the key offers the suite package ohttp speaks. +func Pinned() (relay string, key ohttp.KeyConfig, ok bool) { + return pinned(RelayURL, GatewayKey) +} + +func pinned(relay, key string) (string, ohttp.KeyConfig, bool) { + raw, err := base64.RawURLEncoding.DecodeString(key) + if err != nil || relay == "" || key == "" { + return "", ohttp.KeyConfig{}, false + } + cfg, err := ohttp.ParseKeyConfig(raw) + if err != nil { + return "", ohttp.KeyConfig{}, false + } + u, err := url.Parse(relay) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == energywire.Authority { + return "", ohttp.KeyConfig{}, false + } + return relay, cfg, true +} + +const ( + // The send window after a day closes: late enough that the day's last + // readings are in the ledger, early enough that the gateway still takes + // the date. + firstSend = time.Hour + lastSend = 12 * time.Hour + + sendTimeout = 30 * time.Second + maxResponseBytes = 1 << 10 + + // lastDayKey records the last date this box sent, on the box only, so a + // restart does not send a day twice. + lastDayKey = "energy_report.last_date" +) + +// KV is where the box remembers the last date it sent. +type KV interface { + LoadConfig(key string) (string, bool) + SaveConfig(key, value string) error +} + +// Options wires the sender. +type Options struct { + // Enabled is the owner's answer and the partner rule, read at each send. + Enabled func() bool + Ledger Ledger + KV KV + + // Relay and Key default to Pinned. Tests set them to a fake relay. + Relay string + Key *ohttp.KeyConfig + + Client *http.Client + Logger *slog.Logger + + // Injectable seams. Production leaves them nil. + Now func() time.Time + Random func() float64 + ClockSynced func() bool +} + +// Sender owns the schedule. +type Sender struct { + enabled func() bool + ledger Ledger + kv KV + relay string + key ohttp.KeyConfig + ready bool + client *http.Client + log *slog.Logger + now func() time.Time + random func() float64 + synced func() bool +} + +func New(opts Options) (*Sender, error) { + if opts.Ledger == nil { + return nil, errors.New("energyreport: Ledger is required") + } + s := &Sender{enabled: opts.Enabled, ledger: opts.Ledger, kv: opts.KV, client: opts.Client, log: opts.Logger, + now: opts.Now, random: opts.Random, synced: opts.ClockSynced} + if opts.Key != nil { + s.relay, s.key, s.ready = opts.Relay, *opts.Key, opts.Relay != "" + } else { + s.relay, s.key, s.ready = Pinned() + } + if s.enabled == nil { + s.enabled = func() bool { return false } + } + if s.client == nil { + s.client = &http.Client{Timeout: sendTimeout} + } else { + c := *s.client + s.client = &c + } + // A redirect would resend the sealed report to an address nobody chose. + s.client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + if s.log == nil { + s.log = slog.Default() + } + s.log = s.log.With("component", "energyreport") + if s.now == nil { + s.now = time.Now + } + if s.random == nil { + s.random = randomFloat + } + if s.synced == nil { + s.synced = func() bool { return true } + } + return s, nil +} + +// Preview is what the box would send for yesterday, for Settings. +type Preview struct { + // Relay is where it goes, empty when this build has no relay. + Relay string `json:"relay"` + // Sending is whether this box sends at all: a relay and a gateway key in + // this build, and the owner's answer now. + Sending bool `json:"sending"` + // Day is the closed UTC day it describes. + Day string `json:"day"` + // Payload is the report, or nil when that day is incomplete. + Payload *energywire.Report `json:"payload"` +} + +// Preview builds yesterday's report with the same call the sender uses. +func (s *Sender) Preview(ctx context.Context) Preview { + day := s.now().UTC().Truncate(24*time.Hour).AddDate(0, 0, -1) + p := Preview{Relay: s.relay, Sending: s.ready && s.enabled(), Day: day.Format(time.DateOnly)} + if r, ok, err := Day(ctx, s.ledger, day); err == nil && ok { + p.Payload = &r + } + return p +} + +// Run sends once a day until ctx ends. Without a relay and a gateway key it +// returns at once and sends nothing. +func (s *Sender) Run(ctx context.Context) { + if !s.ready { + return + } + var p plan + for { + now := s.now() + if !p.live(now) { + p = s.draw(now) + } + if !now.Before(p.at) { + s.send(ctx, p.day) + p.done = true + continue + } + // Wake at least hourly, so a wall clock that NTP moves is noticed + // and the plan drawn again. + timer := time.NewTimer(min(p.at.Sub(now), time.Hour)) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +// plan is one day's report: the day it describes and when to send it. +type plan struct { + day time.Time + at time.Time + done bool +} + +// live reports whether the plan still belongs to the clock's day. +func (p plan) live(now time.Time) bool { + return !p.done && !p.day.IsZero() && !now.Before(p.day) && now.Before(p.day.Add(24*time.Hour+lastSend)) +} + +// draw plans the next report. Within a day's send window and before that day +// is sent, the send time is drawn from what is left of the window; otherwise +// from the next day's window. The time is drawn afresh each day, so it does +// not settle into a slot that would name the box. +func (s *Sender) draw(now time.Time) plan { + now = now.UTC() + closed := now.Truncate(24 * time.Hour) + day := closed.AddDate(0, 0, -1) + end := closed.Add(lastSend) + if now.Before(end) && !s.sent(day) { + from := closed.Add(firstSend) + if now.After(from) { + from = now + } + return plan{day: day, at: from.Add(time.Duration(s.random() * float64(end.Sub(from))))} + } + next := closed.AddDate(0, 0, 1) + return plan{day: closed, at: next.Add(firstSend + time.Duration(s.random()*float64(lastSend-firstSend)))} +} + +func (s *Sender) sent(day time.Time) bool { + if s.kv == nil { + return false + } + last, ok := s.kv.LoadConfig(lastDayKey) + return ok && last >= day.Format(time.DateOnly) +} + +// send makes one attempt for one day and forgets about it. There is no +// retry: a lost report is one fewer in a count that says it counts reports. +func (s *Sender) send(ctx context.Context, day time.Time) { + if !s.ready || !s.synced() || !s.enabled() || s.sent(day) { + return + } + r, ok, err := Day(ctx, s.ledger, day) + if err != nil || !ok { + s.log.Debug("no complete day to report", "day", day.Format(time.DateOnly), "err", err) + return + } + // The answer may have changed while the ledger was read. + if !s.enabled() { + return + } + // Remembered before it goes, so a crash mid-send loses the day rather + // than counting it twice. + if s.kv != nil { + if err := s.kv.SaveConfig(lastDayKey, r.Date); err != nil { + return + } + } + sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) + defer cancel() + if err := s.post(sendCtx, r); err != nil { + s.log.Debug("energy report did not go through", "err", err) + } +} + +// post seals one report and reads the gateway's answer. +func (s *Sender) post(ctx context.Context, r energywire.Report) error { + body, err := json.Marshal(r) + if err != nil { + return err + } + inner := ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, + Path: energywire.Path, Content: body}.Marshal(energywire.RequestSize) + sealed, reader, err := ohttp.Encapsulate(s.key, inner) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.relay, bytes.NewReader(sealed)) + if err != nil { + return err + } + req.Header.Set("Content-Type", ohttp.RequestType) + // Pinned, as for the fleet report: the relay sees it, and Go's default + // names the runtime's version. + req.Header.Set("User-Agent", "ftw-energy-report") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil { + return err + } + media, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if resp.StatusCode != http.StatusOK || media != ohttp.ResponseType || len(data) > maxResponseBytes { + return fmt.Errorf("energy report: relay answered %d %q", resp.StatusCode, media) + } + msg, err := reader.Open(data) + if err != nil { + return err + } + answer, err := ohttp.ParseResponse(msg) + if err != nil { + return err + } + if answer.Status != http.StatusNoContent { + return fmt.Errorf("energy report: gateway answered %d", answer.Status) + } + return nil +} diff --git a/go/internal/energyreport/energyreport_test.go b/go/internal/energyreport/energyreport_test.go new file mode 100644 index 000000000..6aebaa1cd --- /dev/null +++ b/go/internal/energyreport/energyreport_test.go @@ -0,0 +1,319 @@ +package energyreport + +import ( + "bytes" + "context" + "crypto/ecdh" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// The release names no relay and no gateway key yet, so it sends nothing. +// Once both are pinned they must be a usable pair. +func TestTheReleasePinsBothOrNeither(t *testing.T) { + _, _, ok := Pinned() + switch { + case RelayURL == "" && GatewayKey == "": + if ok { + t.Fatal("nothing pinned, yet ready") + } + case !ok: + t.Fatalf("the pinned relay %q and gateway key do not make a usable pair", RelayURL) + } +} + +func TestAPinIsRefusedUnlessItIsARelayAndAKey(t *testing.T) { + key := base64.RawURLEncoding.EncodeToString(testGateway(t).Configs()[0].Marshal()) + if _, _, ok := pinned("https://relay.example.net/ftw", key); !ok { + t.Fatal("a good pair was refused") + } + for name, p := range map[string][2]string{ + "no relay": {"", key}, + "no key": {"https://relay.example.net/ftw", ""}, + "plain http": {"http://relay.example.net/ftw", key}, + "FTW Cloud itself": {"https://cloud.ftw.energy/v1/ohttp", key}, + "query": {"https://relay.example.net/ftw?box=1", key}, + "credentials": {"https://a:b@relay.example.net/ftw", key}, + "bad key": {"https://relay.example.net/ftw", "AAAA"}, + "padded base64": {"https://relay.example.net/ftw", key + "="}, + } { + if _, _, ok := pinned(p[0], p[1]); ok { + t.Fatalf("%s was accepted", name) + } + } +} + +func TestTheSendTimeIsDrawnInTheWindowAfterTheDayCloses(t *testing.T) { + closed := day.AddDate(0, 0, 1) + for name, c := range map[string]struct { + now time.Time + random float64 + sent string + wantDay time.Time + from, to time.Time + }{ + "boot before the window": {closed.Add(20 * time.Minute), 0, "", day, closed.Add(time.Hour), closed.Add(time.Hour)}, + "latest in the window": {closed.Add(20 * time.Minute), 1, "", day, closed.Add(12 * time.Hour), closed.Add(12 * time.Hour)}, + "boot inside the window": {closed.Add(6 * time.Hour), 0.5, "", day, closed.Add(9 * time.Hour), closed.Add(9 * time.Hour)}, + "boot after the window": {closed.Add(13 * time.Hour), 0, "", closed, closed.Add(25 * time.Hour), closed.Add(25 * time.Hour)}, + "that day was sent already": {closed.Add(6 * time.Hour), 0, "2026-10-23", closed, closed.Add(25 * time.Hour), closed.Add(25 * time.Hour)}, + "a later day was sent first": {closed.Add(6 * time.Hour), 0, "2026-10-24", closed, closed.Add(25 * time.Hour), closed.Add(25 * time.Hour)}, + } { + kv := memKV{} + if c.sent != "" { + kv[lastDayKey] = c.sent + } + s := newSender(t, Options{KV: kv, Random: func() float64 { return c.random }}) + p := s.draw(c.now) + if !p.day.Equal(c.wantDay) || p.at.Before(c.from) || p.at.After(c.to) || !p.live(c.now) { + t.Fatalf("%s: plan %+v", name, p) + } + } + // A clock that jumps a year makes the plan stale. + p := plan{day: day, at: day.Add(30 * time.Hour)} + if p.live(day.AddDate(1, 0, 0)) || p.live(day.AddDate(-1, 0, 0)) || !p.live(day.Add(30*time.Hour)) { + t.Fatal("a stale plan stayed live") + } +} + +// Box to relay to gateway: the relay sees a fixed-length ciphertext and no +// report, the gateway reads the report and nothing that names the box, and +// the box hears 204. +func TestABoxReportReachesTheGatewayThroughTheRelay(t *testing.T) { + g := testGateway(t) + gw := newFakeGateway(t, g) + relay := newFakeRelay(t, gw.URL) + kv := memKV{} + cfg := g.Configs()[0] + s := newSender(t, Options{Ledger: whole(t), KV: kv, Relay: relay.server.URL, Key: &cfg, Enabled: func() bool { return true }, + Now: func() time.Time { return day.Add(30 * time.Hour) }}) + + s.send(context.Background(), day) + if len(gw.reports) != 1 { + t.Fatalf("gateway got %d reports", len(gw.reports)) + } + got := gw.reports[0] + if got.Date != "2026-10-23" || got.GridImport != 24 || got.VehicleCharge != 24 { + t.Fatalf("report %+v", got) + } + if kv[lastDayKey] != "2026-10-23" || gw.answered != http.StatusNoContent { + t.Fatalf("kv %v answered %d", kv, gw.answered) + } + body := relay.bodies[0] + if bytes.Contains(body, []byte("kwh")) || bytes.Contains(body, []byte("2026-10-23")) || len(body) != 7+32+energywire.RequestSize+16 { + t.Fatalf("the relay saw %d bytes: %q", len(body), body) + } + if relay.headers[0].Get("Content-Type") != ohttp.RequestType || relay.headers[0].Get("User-Agent") != "ftw-energy-report" { + t.Fatalf("relay headers %v", relay.headers[0]) + } + // A restart in the same window does not send the day again. + s.send(context.Background(), day) + if len(gw.reports) != 1 { + t.Fatal("the same day went twice") + } +} + +func TestNothingIsSentWithoutTheOwnersAnswer(t *testing.T) { + g := testGateway(t) + gw := newFakeGateway(t, g) + relay := newFakeRelay(t, gw.URL) + cfg := g.Configs()[0] + ledger := whole(t) + var mu sync.Mutex + reads := 0 + for name, o := range map[string]Options{ + "no answer": {Enabled: nil}, + "the answer is no": {Enabled: func() bool { return false }}, + "clock not set": {Enabled: func() bool { return true }, ClockSynced: func() bool { return false }}, + // Withdrawn while the ledger was being read. + "withdrawn mid-read": {Enabled: func() bool { + mu.Lock() + defer mu.Unlock() + reads++ + return reads == 1 + }}, + "incomplete day": {Enabled: func() bool { return true }, Ledger: wholeDay(t, func(id string, at time.Time) bool { return id == "inverter:1/pv" && at.After(day.Add(time.Hour)) })}, + } { + if o.Ledger == nil { + o.Ledger = ledger + } + o.Relay, o.Key, o.KV = relay.server.URL, &cfg, memKV{} + newSender(t, o).send(context.Background(), day) + if len(relay.bodies) != 0 { + t.Fatalf("%s: sent", name) + } + } +} + +// Without a relay and a gateway key, Run returns and never reads the ledger. +func TestWithoutARelayRunSendsNothing(t *testing.T) { + l := &countingLedger{} + s := newSender(t, Options{Ledger: l, Enabled: func() bool { return true }}) + done := make(chan struct{}) + go func() { s.Run(context.Background()); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run waited with nowhere to send") + } + if l.n != 0 { + t.Fatal("read the ledger with nowhere to send") + } + if p := s.Preview(context.Background()); p.Relay != "" || p.Day == "" { + t.Fatalf("preview %+v", p) + } +} + +// The box believes only a sealed 204 from the gateway it encrypted to. +func TestTheBoxReadsOnlyTheGatewaysSealedAnswer(t *testing.T) { + g := testGateway(t) + cfg := g.Configs()[0] + ledger := whole(t) + r, _, _ := Day(context.Background(), ledger, day) + for name, h := range map[string]http.HandlerFunc{ + "relay error": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) }, + "plain 204": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }, + "forged body": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(make([]byte, 64)) + }, + "sealed 400": sealedAnswer(t, g, http.StatusBadRequest), + "redirect": func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://elsewhere.example/", http.StatusTemporaryRedirect) + }, + } { + srv := httptest.NewServer(h) + s := newSender(t, Options{Ledger: ledger, Relay: srv.URL, Key: &cfg}) + if err := s.post(context.Background(), r); err == nil { + t.Fatalf("%s: accepted", name) + } + srv.Close() + } + srv := httptest.NewServer(sealedAnswer(t, g, http.StatusNoContent)) + defer srv.Close() + s := newSender(t, Options{Ledger: ledger, Relay: srv.URL, Key: &cfg}) + if err := s.post(context.Background(), r); err != nil { + t.Fatal(err) + } +} + +func newSender(t *testing.T, o Options) *Sender { + t.Helper() + if o.Ledger == nil { + o.Ledger = &countingLedger{} + } + s, err := New(o) + if err != nil { + t.Fatal(err) + } + return s +} + +func testGateway(t *testing.T) *ohttp.Gateway { + t.Helper() + k, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + g, err := ohttp.NewGateway(ohttp.Key{ID: 7, Private: k}) + if err != nil { + t.Fatal(err) + } + return g +} + +func sealedAnswer(t *testing.T, g *ohttp.Gateway, status int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _, ctx, err := g.Decapsulate(body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + sealed, _ := ctx.Seal(ohttp.Response{Status: status}.Marshal(energywire.ResponseSize)) + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(sealed) + } +} + +// fakeGateway stands in for FTW Cloud's: it reads only the one inner request. +type fakeGateway struct { + *httptest.Server + g *ohttp.Gateway + reports []energywire.Report + answered int +} + +func newFakeGateway(t *testing.T, g *ohttp.Gateway) *fakeGateway { + f := &fakeGateway{g: g} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + msg, ctx, err := g.Decapsulate(body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + f.answered = http.StatusBadRequest + req, err := ohttp.ParseRequest(msg) + if err == nil && req.Method == energywire.Method && req.Path == energywire.Path && len(msg) == energywire.RequestSize { + if rep, err := energywire.Decode(req.Content, day.Add(30*time.Hour)); err == nil { + f.reports = append(f.reports, rep) + f.answered = http.StatusNoContent + } + } + sealed, _ := ctx.Seal(ohttp.Response{Status: f.answered}.Marshal(energywire.ResponseSize)) + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(sealed) + })) + t.Cleanup(f.Close) + return f +} + +// fakeRelay forwards the body and content type, and nothing about the box. +type fakeRelay struct { + server *httptest.Server + bodies [][]byte + headers []http.Header +} + +func newFakeRelay(t *testing.T, gateway string) *fakeRelay { + f := &fakeRelay{} + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.bodies = append(f.bodies, body) + f.headers = append(f.headers, r.Header.Clone()) + resp, err := http.Post(gateway, r.Header.Get("Content-Type"), bytes.NewReader(body)) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + defer resp.Body.Close() + w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + t.Cleanup(f.server.Close) + return f +} + +type memKV map[string]string + +func (m memKV) LoadConfig(k string) (string, bool) { v, ok := m[k]; return v, ok } +func (m memKV) SaveConfig(k, v string) error { m[k] = v; return nil } + +type countingLedger struct{ n int } + +func (c *countingLedger) EnergyAssets() ([]state.EnergyAsset, error) { c.n++; return nil, nil } +func (c *countingLedger) LoadEnergyHistoryContext(context.Context, state.EnergyHistoryQuery) ([]state.EnergyLedgerPoint, bool, error) { + c.n++ + return nil, false, nil +} diff --git a/go/internal/energyreport/random.go b/go/internal/energyreport/random.go new file mode 100644 index 000000000..ec5eef048 --- /dev/null +++ b/go/internal/energyreport/random.go @@ -0,0 +1,16 @@ +package energyreport + +import ( + "crypto/rand" + "math/big" +) + +// randomFloat draws from the system source, as fleetping's does. +func randomFloat() float64 { + const precision = 1 << 53 + n, err := rand.Int(rand.Reader, big.NewInt(precision)) + if err != nil { + return 0.5 + } + return float64(n.Int64()) / precision +} diff --git a/go/protocol/energywire/energy.go b/go/protocol/energywire/energy.go new file mode 100644 index 000000000..07af21371 --- /dev/null +++ b/go/protocol/energywire/energy.go @@ -0,0 +1,119 @@ +// Package energywire names the anonymous daily energy report: one closed UTC +// day of one home as six whole-kWh sums, and nothing that says whose. +// +// It travels inside Oblivious HTTP (package ohttp) through a relay, so the +// gateway never sees the box's address. It carries no id, version, driver, +// zone or size, because the daily fleet report (package fleetwire) carries +// those, and the two must never be joined into one home's record. +package energywire + +import ( + "bytes" + "encoding/json" + "errors" + "time" +) + +const Schema = "ftw.energy/1" + +// The one inner request a box sends. The gateway answers no other. +const ( + Method = "POST" + Scheme = "https" + Authority = "cloud.ftw.energy" + Path = "/v1/energy/report" +) + +const ( + // RequestSize is every padded inner request, and ResponseSize every + // padded inner response, so the relay sees one length whatever the + // numbers or the answer. + RequestSize = 320 + ResponseSize = 16 + + // MaxBodyBytes bounds the report itself, before any parsing. + MaxBodyBytes = 256 + + // MaxKWh bounds one field of one day. A home on a 3 × 63 A fuse moves + // at most about 1,000 kWh a day. + MaxKWh = 1000 + + // AcceptDays is how many closed days back a report may name. A box sends + // 1 to 12 hours after the day closes, so one day would do; three leave + // room for a slow clock. + AcceptDays = 3 + + // MaxReports caps one date's reports, so junk can add bounded noise and + // no more. + MaxReports = 100_000 +) + +// Fields are the report's sums, in the order the public view lists them. +var Fields = []string{"pv_generation_kwh", "grid_import_kwh", "grid_export_kwh", "battery_charge_kwh", "battery_discharge_kwh", "vehicle_charge_kwh"} + +// Report is one home's closed UTC day. Never add an identifier. +type Report struct { + Schema string `json:"schema"` + Date string `json:"date"` + PVGeneration int64 `json:"pv_generation_kwh"` + GridImport int64 `json:"grid_import_kwh"` + GridExport int64 `json:"grid_export_kwh"` + BatteryCharge int64 `json:"battery_charge_kwh"` + BatteryDischarge int64 `json:"battery_discharge_kwh"` + VehicleCharge int64 `json:"vehicle_charge_kwh"` +} + +// Sums are the report's fields in Fields order. +func (r Report) Sums() []int64 { + return []int64{r.PVGeneration, r.GridImport, r.GridExport, r.BatteryCharge, r.BatteryDischarge, r.VehicleCharge} +} + +// Valid checks the fields a sender fills. The date is checked against the +// receiver's clock in Decode. +func (r Report) Valid() bool { + if r.Schema != Schema || !isDate(r.Date) { + return false + } + for _, v := range r.Sums() { + if v < 0 || v > MaxKWh { + return false + } + } + return true +} + +var errBad = errors.New("energy: invalid report") + +// Decode accepts only the bytes a box writes: the eight fields in their +// order, whole numbers in range, and a date in the last AcceptDays closed UTC +// days. Another order, spacing, a repeated or extra field, or a number +// written another way is refused. +func Decode(data []byte, now time.Time) (Report, error) { + var r Report + if len(data) > MaxBodyBytes || json.Unmarshal(data, &r) != nil || !r.Valid() || !Accepts(r.Date, now) { + return Report{}, errBad + } + if canonical, err := json.Marshal(r); err != nil || !bytes.Equal(canonical, data) { + return Report{}, errBad + } + return r, nil +} + +// Accepts reports whether date is one of the last AcceptDays closed UTC days. +func Accepts(date string, now time.Time) bool { + day, err := time.Parse(time.DateOnly, date) + today := now.UTC().Truncate(24 * time.Hour) + return err == nil && day.Before(today) && !day.Before(today.AddDate(0, 0, -AcceptDays)) +} + +// Closed reports whether no report may name date any more, so its totals are +// final. +func Closed(date string, now time.Time) bool { + day, err := time.Parse(time.DateOnly, date) + return err == nil && day.Before(now.UTC().Truncate(24*time.Hour).AddDate(0, 0, -AcceptDays)) +} + +func isDate(s string) bool { + t, err := time.Parse(time.DateOnly, s) + return err == nil && t.Format(time.DateOnly) == s +} diff --git a/go/protocol/energywire/energy_test.go b/go/protocol/energywire/energy_test.go new file mode 100644 index 000000000..08b60911f --- /dev/null +++ b/go/protocol/energywire/energy_test.go @@ -0,0 +1,92 @@ +package energywire + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +var now = time.Date(2026, 10, 24, 9, 30, 0, 0, time.UTC) + +const valid = `{"schema":"ftw.energy/1","date":"2026-10-23","pv_generation_kwh":12,"grid_import_kwh":30,"grid_export_kwh":4,"battery_charge_kwh":9,"battery_discharge_kwh":8,"vehicle_charge_kwh":0}` + +func TestOnlySixSumsAndADateAreAccepted(t *testing.T) { + r, err := Decode([]byte(valid), now) + if err != nil || r.PVGeneration != 12 || r.GridImport != 30 || r.Date != "2026-10-23" { + t.Fatalf("%+v %v", r, err) + } + for _, bad := range []string{ + strings.Replace(valid, `"ftw.energy/1"`, `"ftw.energy/2"`, 1), + strings.Replace(valid, `2026-10-23`, `2026-10-24`, 1), // today is still open + strings.Replace(valid, `2026-10-23`, `2026-10-20`, 1), // four days back + strings.Replace(valid, `2026-10-23`, `2026-10-32`, 1), + strings.Replace(valid, `2026-10-23`, `2026-1-23`, 1), + strings.Replace(valid, `:12,`, `:12.5,`, 1), + strings.Replace(valid, `:12,`, `:1e1,`, 1), + strings.Replace(valid, `:12,`, `:-1,`, 1), + strings.Replace(valid, `:12,`, `:1001,`, 1), + strings.Replace(valid, `:12,`, `:"12",`, 1), + strings.Replace(valid, `,"vehicle_charge_kwh":0`, ``, 1), + strings.TrimSuffix(valid, "}") + `,"home":"a"}`, + strings.TrimSuffix(valid, "}") + `,"ftw_version":"v1.0.0"}`, + strings.Replace(valid, `"vehicle_charge_kwh":0`, `"vehicle_charge_kwh":0,"vehicle_charge_kwh":0`, 1), + valid + valid, valid + ` {}`, `[]`, `null`, strings.Repeat(" ", MaxBodyBytes) + valid, + } { + if _, err := Decode([]byte(bad), now); err == nil { + t.Fatalf("accepted %s", bad) + } + } +} + +func TestTheReportHasNoFieldThatNamesAHome(t *testing.T) { + var fields map[string]any + b, _ := json.Marshal(Report{}) + if err := json.Unmarshal(b, &fields); err != nil { + t.Fatal(err) + } + if len(fields) != 8 || len(Fields) != 6 { + t.Fatalf("fields %v", fields) + } + for _, name := range append([]string{"schema", "date"}, Fields...) { + if _, ok := fields[name]; !ok { + t.Fatalf("missing %s", name) + } + } +} + +func TestOpenAndClosedDays(t *testing.T) { + for date, want := range map[string][2]bool{ + "2026-10-24": {false, false}, + "2026-10-23": {true, false}, + "2026-10-21": {true, false}, + "2026-10-20": {false, true}, + "2026-09-01": {false, true}, + "2026-10-25": {false, false}, + } { + if got := [2]bool{Accepts(date, now), Closed(date, now)}; got != want { + t.Fatalf("%s: accepts, closed = %v, want %v", date, got, want) + } + } +} + +// The largest report still fits the padded request, so every request has the +// same length on the wire. +func TestTheLargestReportFitsThePaddedRequest(t *testing.T) { + r := Report{Schema: Schema, Date: "2026-10-23", PVGeneration: MaxKWh, GridImport: MaxKWh, GridExport: MaxKWh, + BatteryCharge: MaxKWh, BatteryDischarge: MaxKWh, VehicleCharge: MaxKWh} + body, _ := json.Marshal(r) + if len(body) > MaxBodyBytes { + t.Fatalf("body %d", len(body)) + } + req := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: body}.Marshal(RequestSize) + small := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: []byte(valid)}.Marshal(RequestSize) + if len(req) != RequestSize || len(small) != RequestSize { + t.Fatalf("padded %d and %d", len(req), len(small)) + } + if len(ohttp.Response{Status: 400}.Marshal(ResponseSize)) != ResponseSize { + t.Fatal("response padding") + } +} diff --git a/go/protocol/ohttp/bhttp.go b/go/protocol/ohttp/bhttp.go new file mode 100644 index 000000000..cf6604b41 --- /dev/null +++ b/go/protocol/ohttp/bhttp.go @@ -0,0 +1,200 @@ +package ohttp + +import ( + "encoding/binary" + "errors" +) + +// Known-length Binary HTTP (RFC 9292), the message inside a request and its +// response. Indeterminate-length messages are not read: nothing here sends +// them. + +// ErrMessage is a malformed Binary HTTP message. +var ErrMessage = errors.New("ohttp: invalid binary HTTP message") + +// Field is one header or trailer line. +type Field struct { + Name, Value string +} + +// Request is a known-length request. Trailers are always empty. +type Request struct { + Method, Scheme, Authority, Path string + Fields []Field + Content []byte +} + +// Response is a known-length final response. Trailers are always empty, and +// informational responses are not carried. +type Response struct { + Status int + Fields []Field + Content []byte +} + +const ( + knownRequest = 0 + knownResponse = 1 +) + +// Marshal writes the request, then zero padding up to size bytes. Padding +// keeps every request the same length on the wire (RFC 9292, section 3.8). +func (r Request) Marshal(size int) []byte { + b := appendVarint(nil, knownRequest) + for _, s := range []string{r.Method, r.Scheme, r.Authority, r.Path} { + b = appendBytes(b, []byte(s)) + } + b = appendFields(b, r.Fields) + b = appendBytes(b, r.Content) + b = appendVarint(b, 0) + return pad(b, size) +} + +// Marshal writes the response, then zero padding up to size bytes. +func (r Response) Marshal(size int) []byte { + b := appendVarint(nil, knownResponse) + b = appendVarint(b, uint64(r.Status)) + b = appendFields(b, r.Fields) + b = appendBytes(b, r.Content) + b = appendVarint(b, 0) + return pad(b, size) +} + +// ParseRequest reads a known-length request. Sections after the control data +// may be truncated, and anything after the trailers must be zero padding. +func ParseRequest(b []byte) (Request, error) { + d := decoder{b: b} + if d.varint() != knownRequest { + return Request{}, ErrMessage + } + var r Request + r.Method, r.Scheme, r.Authority, r.Path = string(d.bytes()), string(d.bytes()), string(d.bytes()), string(d.bytes()) + if d.err != nil { + return Request{}, ErrMessage + } + r.Fields = d.fields() + r.Content = d.optional() + trailers := d.fields() + if d.err != nil || len(trailers) != 0 || !d.padding() { + return Request{}, ErrMessage + } + return r, nil +} + +// ParseResponse reads a known-length final response. +func ParseResponse(b []byte) (Response, error) { + d := decoder{b: b} + if d.varint() != knownResponse { + return Response{}, ErrMessage + } + status := d.varint() + if d.err != nil || status < 200 || status > 599 { + return Response{}, ErrMessage + } + r := Response{Status: int(status)} + r.Fields = d.fields() + r.Content = d.optional() + trailers := d.fields() + if d.err != nil || len(trailers) != 0 || !d.padding() { + return Response{}, ErrMessage + } + return r, nil +} + +func appendVarint(b []byte, v uint64) []byte { + switch { + case v < 1<<6: + return append(b, byte(v)) + case v < 1<<14: + return binary.BigEndian.AppendUint16(b, uint16(v)|0x4000) + case v < 1<<30: + return binary.BigEndian.AppendUint32(b, uint32(v)|0x8000_0000) + default: + return binary.BigEndian.AppendUint64(b, v|0xc000_0000_0000_0000) + } +} + +func appendBytes(b, v []byte) []byte { + return append(appendVarint(b, uint64(len(v))), v...) +} + +func appendFields(b []byte, fields []Field) []byte { + var lines []byte + for _, f := range fields { + lines = appendBytes(lines, []byte(f.Name)) + lines = appendBytes(lines, []byte(f.Value)) + } + return appendBytes(b, lines) +} + +func pad(b []byte, size int) []byte { + if len(b) < size { + b = append(b, make([]byte, size-len(b))...) + } + return b +} + +type decoder struct { + b []byte + err error +} + +func (d *decoder) varint() uint64 { + if d.err != nil || len(d.b) == 0 { + d.err = ErrMessage + return 0 + } + n := 1 << (d.b[0] >> 6) + if len(d.b) < n { + d.err = ErrMessage + return 0 + } + v := uint64(d.b[0] & 0x3f) + for _, c := range d.b[1:n] { + v = v<<8 | uint64(c) + } + d.b = d.b[n:] + return v +} + +func (d *decoder) bytes() []byte { + n := d.varint() + if d.err != nil || n > uint64(len(d.b)) { + d.err = ErrMessage + return nil + } + v := d.b[:n] + d.b = d.b[n:] + return v +} + +// optional reads a length-prefixed part that a truncated message leaves out. +func (d *decoder) optional() []byte { + if d.err != nil || len(d.b) == 0 { + return nil + } + return d.bytes() +} + +func (d *decoder) fields() []Field { + section := decoder{b: d.optional()} + var out []Field + for d.err == nil && len(section.b) > 0 { + name, value := section.bytes(), section.bytes() + if section.err != nil || len(name) == 0 { + d.err = ErrMessage + return nil + } + out = append(out, Field{Name: string(name), Value: string(value)}) + } + return out +} + +func (d *decoder) padding() bool { + for _, c := range d.b { + if c != 0 { + return false + } + } + return true +} diff --git a/go/protocol/ohttp/bhttp_test.go b/go/protocol/ohttp/bhttp_test.go new file mode 100644 index 000000000..f3bd9acc9 --- /dev/null +++ b/go/protocol/ohttp/bhttp_test.go @@ -0,0 +1,63 @@ +package ohttp + +import ( + "bytes" + "encoding/hex" + "testing" +) + +func TestBinaryHTTPRoundTripsWithPadding(t *testing.T) { + req := Request{Method: "POST", Scheme: "https", Authority: "cloud.ftw.energy", Path: "/v1/x", + Fields: []Field{{"content-type", "application/json"}}, Content: bytes.Repeat([]byte{'a'}, 100)} + b := req.Marshal(256) + if len(b) != 256 { + t.Fatalf("padded to %d", len(b)) + } + got, err := ParseRequest(b) + if err != nil || got.Method != req.Method || got.Path != req.Path || len(got.Fields) != 1 || got.Fields[0] != req.Fields[0] || !bytes.Equal(got.Content, req.Content) { + t.Fatalf("%+v %v", got, err) + } + // Content longer than one byte's varint, and no padding asked for. + long := Request{Method: "POST", Scheme: "https", Authority: "a", Path: "/", Content: make([]byte, 20000)} + if got, err := ParseRequest(long.Marshal(0)); err != nil || len(got.Content) != 20000 { + t.Fatalf("long content %d %v", len(got.Content), err) + } + + resp := Response{Status: 204}.Marshal(16) + if r, err := ParseResponse(resp); err != nil || r.Status != 204 || len(resp) != 16 { + t.Fatalf("response %+v %v", r, err) + } +} + +func TestBinaryHTTPRefusesMalformedMessages(t *testing.T) { + good := Request{Method: "POST", Scheme: "https", Authority: "a", Path: "/", Content: []byte("x")}.Marshal(0) + withTrailer := append(bytes.Clone(good[:len(good)-1]), 4, 1, 'a', 1, 'b') + for name, b := range map[string][]byte{ + "empty": nil, + "response": Response{Status: 200}.Marshal(0), + "indeterminate": {2}, + "cut control": good[:4], + "cut content": good[:len(good)-2], + "bad padding": append(bytes.Clone(good), 0, 1), + "trailers": withTrailer, + "long length": {0, 0x3f}, + } { + if _, err := ParseRequest(b); err == nil { + t.Fatalf("%s parsed", name) + } + } + for name, b := range map[string][]byte{ + "informational": unhexB("014064"), + "status 600": unhexB("014258"), + "request": good, + } { + if _, err := ParseResponse(b); err == nil { + t.Fatalf("%s parsed", name) + } + } +} + +func unhexB(s string) []byte { + b, _ := hex.DecodeString(s) + return b +} diff --git a/go/protocol/ohttp/ohttp.go b/go/protocol/ohttp/ohttp.go new file mode 100644 index 000000000..cb990d5c4 --- /dev/null +++ b/go/protocol/ohttp/ohttp.go @@ -0,0 +1,335 @@ +// Package ohttp is Oblivious HTTP (RFC 9458) for one suite: DHKEM(X25519, +// HKDF-SHA256), HKDF-SHA256 and AES-128-GCM, with the known-length Binary +// HTTP messages (RFC 9292) it carries. +// +// A box sends its anonymous daily energy report this way: sealed to FTW +// Cloud's gateway key and posted to an independent relay. The relay sees the +// box's address and a ciphertext of fixed size. The gateway sees the report +// and the relay's address. Neither sees both. +// +// HPKE is the standard library's crypto/hpke. This package adds only the +// framing RFC 9458 puts around it and the response key schedule, and it +// speaks one suite so that no message can pick a weaker one. +package ohttp + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/ecdh" + "crypto/hkdf" + "crypto/hpke" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "slices" +) + +// The one suite, by its HPKE ids. +const ( + KEMX25519 uint16 = 0x0020 + KDFHKDFSHA256 uint16 = 0x0001 + AEADAES128GCM uint16 = 0x0001 +) + +// Media types of RFC 9458, section 9. +const ( + RequestType = "message/ohttp-req" + ResponseType = "message/ohttp-res" + KeysType = "application/ohttp-keys" +) + +const ( + // keyLen is Npk and Nenc for X25519. + keyLen = 32 + // headerLen is the request header: key id, KEM, KDF and AEAD. + headerLen = 7 + // The response key schedule for AES-128-GCM: Nk 16, Nn 12, and a + // response nonce of max(Nn, Nk). + aeadKeyLen = 16 + aeadNonceLen = 12 + nonceLen = 16 + tagLen = 16 + + requestLabel = "message/bhttp request" + responseLabel = "message/bhttp response" +) + +var ( + ErrConfig = errors.New("ohttp: invalid key configuration") + ErrUnsupported = errors.New("ohttp: key configuration offers no supported suite") + ErrRequest = errors.New("ohttp: cannot decapsulate the request") + ErrResponse = errors.New("ohttp: cannot open the response") +) + +// Algorithms is one KDF and AEAD pair a key configuration offers. +type Algorithms struct { + KDF, AEAD uint16 +} + +var suite = Algorithms{KDF: KDFHKDFSHA256, AEAD: AEADAES128GCM} + +// KeyConfig is a gateway's public key and what it accepts with it +// (RFC 9458, section 3). +type KeyConfig struct { + ID uint8 + KEM uint16 + PublicKey []byte + Algorithms []Algorithms +} + +// Marshal writes the configuration as RFC 9458 section 3.1 lays it out. +func (c KeyConfig) Marshal() []byte { + b := make([]byte, 0, 3+len(c.PublicKey)+2+4*len(c.Algorithms)) + b = append(b, c.ID) + b = binary.BigEndian.AppendUint16(b, c.KEM) + b = append(b, c.PublicKey...) + b = binary.BigEndian.AppendUint16(b, uint16(4*len(c.Algorithms))) + for _, a := range c.Algorithms { + b = binary.BigEndian.AppendUint16(b, a.KDF) + b = binary.BigEndian.AppendUint16(b, a.AEAD) + } + return b +} + +// ParseKeyConfig reads exactly one configuration. Only the X25519 KEM is +// known, because a key length cannot be read for a KEM this package does not +// speak. +func ParseKeyConfig(b []byte) (KeyConfig, error) { + if len(b) < 3 { + return KeyConfig{}, ErrConfig + } + c := KeyConfig{ID: b[0], KEM: binary.BigEndian.Uint16(b[1:3])} + if c.KEM != KEMX25519 { + return KeyConfig{}, ErrUnsupported + } + b = b[3:] + if len(b) < keyLen+2 { + return KeyConfig{}, ErrConfig + } + c.PublicKey = bytes.Clone(b[:keyLen]) + n := int(binary.BigEndian.Uint16(b[keyLen:])) + b = b[keyLen+2:] + if n < 4 || n%4 != 0 || len(b) != n { + return KeyConfig{}, ErrConfig + } + for ; len(b) > 0; b = b[4:] { + c.Algorithms = append(c.Algorithms, Algorithms{KDF: binary.BigEndian.Uint16(b), AEAD: binary.BigEndian.Uint16(b[2:])}) + } + if _, err := hpke.DHKEM(ecdh.X25519()).NewPublicKey(c.PublicKey); err != nil { + return KeyConfig{}, ErrConfig + } + return c, nil +} + +// MarshalKeys writes application/ohttp-keys: each configuration after its +// two-byte length. +func MarshalKeys(configs ...KeyConfig) []byte { + var b []byte + for _, c := range configs { + m := c.Marshal() + b = binary.BigEndian.AppendUint16(b, uint16(len(m))) + b = append(b, m...) + } + return b +} + +// ParseKeys reads application/ohttp-keys. A configuration for another KEM is +// skipped, as RFC 9458 section 3.2 allows; a malformed one fails the whole +// list. +func ParseKeys(b []byte) ([]KeyConfig, error) { + var out []KeyConfig + for len(b) > 0 { + if len(b) < 2 { + return nil, ErrConfig + } + n := int(binary.BigEndian.Uint16(b)) + if len(b) < 2+n { + return nil, ErrConfig + } + c, err := ParseKeyConfig(b[2 : 2+n]) + switch { + case errors.Is(err, ErrUnsupported): + case err != nil: + return nil, err + default: + out = append(out, c) + } + b = b[2+n:] + } + return out, nil +} + +func header(id uint8) []byte { + h := []byte{id} + h = binary.BigEndian.AppendUint16(h, KEMX25519) + h = binary.BigEndian.AppendUint16(h, suite.KDF) + return binary.BigEndian.AppendUint16(h, suite.AEAD) +} + +func info(hdr []byte) []byte { + return append(append([]byte(requestLabel), 0), hdr...) +} + +// ClientContext opens the one response to one request. +type ClientContext struct { + enc []byte + sender *hpke.Sender +} + +// Encapsulate seals msg to the gateway that published cfg (RFC 9458, +// section 4.3). A configuration that does not offer this package's suite is +// refused, never downgraded. +func Encapsulate(cfg KeyConfig, msg []byte) ([]byte, *ClientContext, error) { + if cfg.KEM != KEMX25519 || !slices.Contains(cfg.Algorithms, suite) { + return nil, nil, ErrUnsupported + } + pk, err := hpke.DHKEM(ecdh.X25519()).NewPublicKey(cfg.PublicKey) + if err != nil { + return nil, nil, ErrConfig + } + hdr := header(cfg.ID) + enc, sender, err := hpke.NewSender(pk, hpke.HKDFSHA256(), hpke.AES128GCM(), info(hdr)) + if err != nil { + return nil, nil, err + } + ct, err := sender.Seal(nil, msg) + if err != nil { + return nil, nil, err + } + out := slices.Concat(hdr, enc, ct) + return out, &ClientContext{enc: enc, sender: sender}, nil +} + +// Open reads the gateway's answer (RFC 9458, section 4.4). +func (c *ClientContext) Open(encResp []byte) ([]byte, error) { + secret, err := c.sender.Export(responseLabel, nonceLen) + if err != nil { + return nil, ErrResponse + } + return openResponse(secret, c.enc, encResp) +} + +// Key is one private key a gateway decapsulates with, under its key id. +type Key struct { + ID uint8 + Private *ecdh.PrivateKey +} + +// Gateway decapsulates requests sealed to its keys. +type Gateway struct { + keys map[uint8]hpke.PrivateKey + configs []KeyConfig +} + +// NewGateway holds X25519 keys under distinct ids. Each is published with +// this package's one suite. +func NewGateway(keys ...Key) (*Gateway, error) { + g := &Gateway{keys: map[uint8]hpke.PrivateKey{}} + for _, k := range keys { + if k.Private == nil || k.Private.Curve() != ecdh.X25519() { + return nil, ErrConfig + } + if _, dup := g.keys[k.ID]; dup { + return nil, ErrConfig + } + priv, err := hpke.NewDHKEMPrivateKey(k.Private) + if err != nil { + return nil, ErrConfig + } + g.keys[k.ID] = priv + g.configs = append(g.configs, KeyConfig{ID: k.ID, KEM: KEMX25519, PublicKey: k.Private.PublicKey().Bytes(), Algorithms: []Algorithms{suite}}) + } + if len(g.keys) == 0 { + return nil, ErrConfig + } + return g, nil +} + +// Configs are the gateway's key configurations, in the order given. +func (g *Gateway) Configs() []KeyConfig { return slices.Clone(g.configs) } + +// ServerContext seals the one response to one request. +type ServerContext struct { + enc, secret []byte +} + +// Decapsulate opens a request (RFC 9458, section 4.3). Every failure is the +// same error, so an answer says nothing about which check refused it. +func (g *Gateway) Decapsulate(req []byte) ([]byte, *ServerContext, error) { + if len(req) < headerLen+keyLen+tagLen { + return nil, nil, ErrRequest + } + key, ok := g.keys[req[0]] + if !ok || !bytes.Equal(req[:headerLen], header(req[0])) { + return nil, nil, ErrRequest + } + hdr, enc, ct := req[:headerLen], req[headerLen:headerLen+keyLen], req[headerLen+keyLen:] + r, err := hpke.NewRecipient(enc, key, hpke.HKDFSHA256(), hpke.AES128GCM(), info(hdr)) + if err != nil { + return nil, nil, ErrRequest + } + msg, err := r.Open(nil, ct) + if err != nil { + return nil, nil, ErrRequest + } + secret, err := r.Export(responseLabel, nonceLen) + if err != nil { + return nil, nil, ErrRequest + } + return msg, &ServerContext{enc: bytes.Clone(enc), secret: secret}, nil +} + +// Seal encapsulates the response under a fresh nonce (RFC 9458, section 4.4). +func (s *ServerContext) Seal(resp []byte) ([]byte, error) { + nonce := make([]byte, nonceLen) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + return sealResponse(s.secret, s.enc, nonce, resp) +} + +func responseAEAD(secret, enc, nonce []byte) (cipher.AEAD, []byte, error) { + prk, err := hkdf.Extract(sha256.New, secret, slices.Concat(enc, nonce)) + if err != nil { + return nil, nil, err + } + key, err := hkdf.Expand(sha256.New, prk, "key", aeadKeyLen) + if err != nil { + return nil, nil, err + } + iv, err := hkdf.Expand(sha256.New, prk, "nonce", aeadNonceLen) + if err != nil { + return nil, nil, err + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, nil, err + } + aead, err := cipher.NewGCM(block) + return aead, iv, err +} + +func sealResponse(secret, enc, nonce, resp []byte) ([]byte, error) { + aead, iv, err := responseAEAD(secret, enc, nonce) + if err != nil { + return nil, err + } + return aead.Seal(bytes.Clone(nonce), iv, resp, nil), nil +} + +func openResponse(secret, enc, encResp []byte) ([]byte, error) { + if len(encResp) < nonceLen+tagLen { + return nil, ErrResponse + } + aead, iv, err := responseAEAD(secret, enc, encResp[:nonceLen]) + if err != nil { + return nil, ErrResponse + } + msg, err := aead.Open(nil, iv, encResp[nonceLen:], nil) + if err != nil { + return nil, ErrResponse + } + return msg, nil +} diff --git a/go/protocol/ohttp/ohttp_test.go b/go/protocol/ohttp/ohttp_test.go new file mode 100644 index 000000000..12f12386d --- /dev/null +++ b/go/protocol/ohttp/ohttp_test.go @@ -0,0 +1,247 @@ +package ohttp + +import ( + "bytes" + "crypto/ecdh" + "encoding/hex" + "errors" + "strings" + "testing" +) + +// RFC 9458, Appendix A. +const ( + rfcSecretKey = "3c168975674b2fa8e465970b79c8dcf09f1c741626480bd4c6162fc5b6a98e1a" + rfcKeyConfig = "01002031e1f05a740102115220e9af918f738674aec95f54db6e04eb705aae8e79815500080001000100010003" + rfcRequest = "00034745540568747470730b6578616d706c652e636f6d012f" + rfcInfo = "6d6573736167652f626874747020726571756573740001002000010001" + rfcEncRequest = "010020000100014b28f881333e7c164ffc499ad9796f877f4e1051ee6d31bad19dec96c208b4726374e469135906992e1268c594d2a10c695d858c40a026e7965e7d86b83dd440b2c0185204b4d63525" + rfcResponse = "0140c8" + rfcSecret = "62d87a6ba569ee81014c2641f52bea36" + rfcNonce = "c789e7151fcba46158ca84b04464910d" + rfcEncResp = "c789e7151fcba46158ca84b04464910d86f9013e404feea014e7be4a441f234f857fbd" +) + +func unhex(t *testing.T, s string) []byte { + t.Helper() + b, err := hex.DecodeString(s) + if err != nil { + t.Fatal(err) + } + return b +} + +func rfcGateway(t *testing.T) *Gateway { + t.Helper() + priv, err := ecdh.X25519().NewPrivateKey(unhex(t, rfcSecretKey)) + if err != nil { + t.Fatal(err) + } + g, err := NewGateway(Key{ID: 1, Private: priv}) + if err != nil { + t.Fatal(err) + } + return g +} + +func TestRFC9458KeyConfiguration(t *testing.T) { + cfg, err := ParseKeyConfig(unhex(t, rfcKeyConfig)) + if err != nil { + t.Fatal(err) + } + if cfg.ID != 1 || cfg.KEM != KEMX25519 || len(cfg.Algorithms) != 2 || + cfg.Algorithms[0] != (Algorithms{KDFHKDFSHA256, AEADAES128GCM}) || cfg.Algorithms[1] != (Algorithms{KDFHKDFSHA256, 0x0003}) { + t.Fatalf("parsed %+v", cfg) + } + if got := hex.EncodeToString(cfg.Marshal()); got != rfcKeyConfig { + t.Fatalf("marshal %s", got) + } + if got := rfcGateway(t).Configs()[0]; !bytes.Equal(got.PublicKey, cfg.PublicKey) || got.ID != 1 { + t.Fatalf("the secret key does not give the published key: %+v", got) + } + if got := hex.EncodeToString(info(header(1))); got != rfcInfo { + t.Fatalf("info %s", got) + } +} + +// The gateway opens the RFC's request, exports its secret, and seals the +// RFC's response byte for byte under the RFC's nonce. +func TestRFC9458GatewayVectors(t *testing.T) { + msg, ctx, err := rfcGateway(t).Decapsulate(unhex(t, rfcEncRequest)) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(msg) != rfcRequest { + t.Fatalf("request %x", msg) + } + if hex.EncodeToString(ctx.secret) != rfcSecret { + t.Fatalf("secret %x", ctx.secret) + } + resp, err := sealResponse(ctx.secret, ctx.enc, unhex(t, rfcNonce), unhex(t, rfcResponse)) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(resp) != rfcEncResp { + t.Fatalf("response %x", resp) + } + r, err := ParseRequest(msg) + if err != nil || r.Method != "GET" || r.Scheme != "https" || r.Authority != "example.com" || r.Path != "/" || len(r.Fields) != 0 || len(r.Content) != 0 { + t.Fatalf("binary request %+v %v", r, err) + } +} + +// The client half: its request header is the RFC's, and its response key +// schedule opens the RFC's encapsulated response. +func TestRFC9458ClientVectors(t *testing.T) { + enc := unhex(t, rfcEncRequest) + if got := hex.EncodeToString(header(1)); got != hex.EncodeToString(enc[:headerLen]) { + t.Fatalf("header %s", got) + } + resp, err := openResponse(unhex(t, rfcSecret), enc[headerLen:headerLen+keyLen], unhex(t, rfcEncResp)) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(resp) != rfcResponse { + t.Fatalf("response %x", resp) + } + r, err := ParseResponse(resp) + if err != nil || r.Status != 200 { + t.Fatalf("binary response %+v %v", r, err) + } + + cfg, err := ParseKeyConfig(unhex(t, rfcKeyConfig)) + if err != nil { + t.Fatal(err) + } + req, cctx, err := Encapsulate(cfg, unhex(t, rfcRequest)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(req[:headerLen], enc[:headerLen]) || len(req) != len(enc) { + t.Fatalf("request %x", req) + } + msg, sctx, err := rfcGateway(t).Decapsulate(req) + if err != nil || hex.EncodeToString(msg) != rfcRequest { + t.Fatalf("gateway read %x %v", msg, err) + } + sealed, err := sctx.Seal(unhex(t, rfcResponse)) + if err != nil { + t.Fatal(err) + } + got, err := cctx.Open(sealed) + if err != nil || hex.EncodeToString(got) != rfcResponse { + t.Fatalf("client read %x %v", got, err) + } +} + +func TestTheGatewayRefusesWhatItCannotOpen(t *testing.T) { + g := rfcGateway(t) + good := unhex(t, rfcEncRequest) + flip := func(i int) []byte { + b := bytes.Clone(good) + b[i] ^= 1 + return b + } + for name, req := range map[string][]byte{ + "empty": nil, + "short": good[:headerLen+keyLen], + "key id": flip(0), + "kem": flip(2), + "kdf": flip(4), + "aead": flip(6), + "enc": flip(headerLen), + "ciphertext": flip(len(good) - 1), + "truncated": good[:len(good)-1], + "extra bytes": append(bytes.Clone(good), 0), + } { + if _, _, err := g.Decapsulate(req); !errors.Is(err, ErrRequest) { + t.Fatalf("%s: %v", name, err) + } + } +} + +// A request sealed to one key cannot be read by a gateway holding another, +// even under the same key id. A box encrypts only to the configuration its +// release pins, so a key swapped in at the service reads nothing. +func TestASwappedKeyReadsNothing(t *testing.T) { + cfg, err := ParseKeyConfig(unhex(t, rfcKeyConfig)) + if err != nil { + t.Fatal(err) + } + req, _, err := Encapsulate(cfg, []byte("report")) + if err != nil { + t.Fatal(err) + } + other, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + g, err := NewGateway(Key{ID: 1, Private: other}) + if err != nil { + t.Fatal(err) + } + if _, _, err := g.Decapsulate(req); !errors.Is(err, ErrRequest) { + t.Fatalf("a different key opened the request: %v", err) + } +} + +func TestTheClientRefusesOtherSuitesAndForgedResponses(t *testing.T) { + cfg, err := ParseKeyConfig(unhex(t, rfcKeyConfig)) + if err != nil { + t.Fatal(err) + } + only := cfg + only.Algorithms = []Algorithms{{KDFHKDFSHA256, 0x0003}} + if _, _, err := Encapsulate(only, nil); !errors.Is(err, ErrUnsupported) { + t.Fatalf("ChaCha20 only: %v", err) + } + _, cctx, err := Encapsulate(cfg, []byte("x")) + if err != nil { + t.Fatal(err) + } + for _, forged := range [][]byte{nil, make([]byte, nonceLen+tagLen), unhex(t, rfcEncResp)} { + if _, err := cctx.Open(forged); !errors.Is(err, ErrResponse) { + t.Fatalf("opened %x: %v", forged, err) + } + } +} + +func TestKeyConfigurationsAreReadStrictly(t *testing.T) { + good := unhex(t, rfcKeyConfig) + for name, b := range map[string][]byte{ + "empty": nil, + "trailing byte": append(bytes.Clone(good), 0), + "short key": append(bytes.Clone(good[:3+31]), good[3+32:]...), + "no algorithms": append(bytes.Clone(good[:3+32]), 0, 0), + "odd length": append(bytes.Clone(good[:3+32]), 0, 6, 0, 1, 0, 1, 0, 1), + } { + if _, err := ParseKeyConfig(b); err == nil { + t.Fatalf("%s parsed", name) + } + } + p256 := bytes.Clone(good) + p256[2] = 0x10 + if _, err := ParseKeyConfig(p256); !errors.Is(err, ErrUnsupported) { + t.Fatalf("P-256: %v", err) + } + + cfg, _ := ParseKeyConfig(good) + list := MarshalKeys(KeyConfig{ID: 9, KEM: 0x0010, PublicKey: make([]byte, 65), Algorithms: []Algorithms{{1, 1}}}, cfg) + got, err := ParseKeys(list) + if err != nil || len(got) != 1 || !bytes.Equal(got[0].Marshal(), good) { + t.Fatalf("keys %+v %v", got, err) + } + if _, err := ParseKeys(list[:len(list)-1]); err == nil { + t.Fatal("a cut list parsed") + } + if _, err := NewGateway(); err == nil { + t.Fatal("a gateway without keys") + } + k, _ := ecdh.X25519().GenerateKey(nil) + if _, err := NewGateway(Key{ID: 1, Private: k}, Key{ID: 1, Private: k}); err == nil { + t.Fatal("two keys under one id") + } + if !strings.HasPrefix(RequestType, "message/") || KeysType != "application/ohttp-keys" { + t.Fatal("media types") + } +} diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs index 1d3749784..6a5824f1e 100644 --- a/web/fleet-ping-tab.test.mjs +++ b/web/fleet-ping-tab.test.mjs @@ -271,3 +271,50 @@ describe("payloadRows", () => { assert.equal(payloadRows("not an object").length, 0); }); }); + +describe("the daily energy report", () => { + const { energyRows, energyLine } = tab._pure; + const energy = { + schema: "ftw.energy/1", date: "2026-10-23", pv_generation_kwh: 12, grid_import_kwh: 30, grid_export_kwh: 4, + battery_charge_kwh: 9, battery_discharge_kwh: 8, vehicle_charge_kwh: 0, + }; + + it("shows every field the report carries, in kWh", () => { + const rows = energyRows(energy); + const byKey = Object.fromEntries(rows.map((r) => [r.key, r])); + assert.equal(rows.length, Object.keys(energy).length - 1); + assert.equal(byKey.pv_generation_kwh.value, "12 kWh"); + assert.equal(byKey.vehicle_charge_kwh.label, "Vehicle charge"); + assert.equal(byKey.date.value, "2026-10-23"); + const extra = energyRows({ ...energy, home: "x" }).find((r) => r.key === "home"); + assert.ok(extra, "an unexpected field was silently dropped"); + }); + + it("says nothing is sent from a release without a relay", () => { + const line = energyLine({ relay: "", sending: false, day: "2026-10-23", payload: energy }); + assert.match(line, /no relay yet/i); + assert.doesNotMatch(line, /^Sent/); + }); + + it("does not point at a report it is not showing", () => { + assert.doesNotMatch(energyLine({ relay: "", day: "2026-10-23", payload: null }), /this is the report/i); + assert.doesNotMatch(energyLine({ relay: "https://relay.example.net/ftw", day: "2026-10-23", payload: null }), /this is the report/i); + }); + + it("says who sees what while it is sent, and nothing is sent while it is off", () => { + const relay = "https://relay.example.net/ftw"; + const on = energyLine({ relay, sending: true, payload: energy }); + assert.match(on, /^Sent through https:\/\/relay\.example\.net/); + assert.match(on, /sees this box's address but not the report/); + assert.match(on, /reads the report but not the address/); + assert.match(energyLine({ relay, sending: false, payload: energy }), /^Nothing is sent/); + }); + + it("names the fields, the missing id and the relay in its paragraph", () => { + const html = render({}); + assert.match(html, /id="fleet-energy-payload"/); + for (const want of [/solar generation/i, /grid import and export/i, /battery charge/i, /vehicle charge/i, /whole kWh/i, /no id/i, /independent relay/i, /skipped/i]) { + assert.match(html, want, `the paragraph does not say ${want}`); + } + }); +}); diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 4ed970d92..080cc2671 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -91,6 +91,49 @@ ", once a day, if you switched it on." + handling; } + // Labels for the daily energy report. As above, a field with no label is + // shown under its own name. + var ENERGY_LABELS = { + date: "Day (UTC)", + pv_generation_kwh: "Solar generation", + grid_import_kwh: "Grid import", + grid_export_kwh: "Grid export", + battery_charge_kwh: "Battery charge", + battery_discharge_kwh: "Battery discharge", + vehicle_charge_kwh: "Vehicle charge", + }; + + // energyRows is payloadRows for the energy report: driven by its keys, so + // nothing it carries can be missing from this screen. + function energyRows(payload) { + if (!payload || typeof payload !== "object") return []; + return Object.keys(payload) + .filter(function (key) { return key !== "schema"; }) + .map(function (key) { + var value = payload[key]; + var text = /_kwh$/.test(key) && typeof value === "number" ? value + " kWh" : String(value); + return { key: key, label: ENERGY_LABELS[key] || key, value: text }; + }); + } + + // energyLine is whereLine for the energy report. It follows the box's own + // answer, and says so when this release has no relay to send through. + // + // Without a report to show, it does not point at one. + function energyLine(energy) { + var shown = !!(energy && energy.payload); + if (!energy || !energy.relay) { + return "This release has no relay yet, so no energy report is sent." + + (shown ? " This is the report it would send." : ""); + } + var how = " The relay sees this box's address but not the report. FTW Cloud reads the report but not the address, adds it to that day's totals and keeps nothing of it."; + if (energy.sending) { + return "Sent through " + energy.relay + ", an independent relay, once a day, 1 to 12 hours after the UTC day ends." + how; + } + return "Nothing is sent. " + (shown ? "This is the report that would go" : "A report would go") + " through " + + energy.relay + ", an independent relay, once a day, if sharing were on." + how; + } + function row(label, value) { var el = document.createElement("div"); el.className = "fleet-ping-row"; @@ -134,6 +177,36 @@ slot.appendChild(exact); } + function paintEnergy(energy) { + var slot = document.getElementById("fleet-energy-payload"); + if (!slot) return; + slot.textContent = ""; + var rows = energyRows(energy && energy.payload); + var note = document.createElement("p"); + note.className = "hint fleet-energy-note"; + if (!energy) { + note.textContent = "This box has no energy report."; + slot.appendChild(note); + return; + } + if (rows.length === 0) { + note.textContent = "The energy record for " + energy.day + " has a gap, so this box has no report for that day. A day with a gap is skipped, never sent in part."; + slot.appendChild(note); + } + rows.forEach(function (r) { slot.appendChild(row(r.label, r.value)); }); + + var where = document.createElement("p"); + where.className = "hint fleet-energy-note"; + where.textContent = energyLine(energy); + slot.appendChild(where); + + if (rows.length === 0) return; + var exact = document.createElement("pre"); + exact.className = "fleet-ping-json"; + exact.textContent = JSON.stringify(energy.payload, null, 2); + slot.appendChild(exact); + } + function refresh() { var slot = document.getElementById("fleet-ping-payload"); if (!slot) return; @@ -149,6 +222,7 @@ return; } paint(body); + paintEnergy(body.energy); }) .catch(function () { slot.textContent = ""; @@ -164,7 +238,7 @@ // app relay. Keep this as a reusable panel instead of a tab so the payload // and privacy wording stay in one place. S.fleetPing = { - _pure: { payloadRows: payloadRows, describe: describe, whereLine: whereLine }, + _pure: { payloadRows: payloadRows, describe: describe, whereLine: whereLine, energyRows: energyRows, energyLine: energyLine }, // The line under the payload is the box's answer, not the checkbox's, so // it can only be right again once the box has the new config. Saving is @@ -183,7 +257,7 @@ return ( "
Anonymous daily report" + '

The project statistics choice above controls whether this box contributes. This additional daily report describes versions and integrations. It does not count unique boxes.

' + - '

When sharing is on, this box sends six things once a day: its ' + + '

When sharing is on, this report carries six things once a day: its ' + "FTW version, its release channel, the kinds of device it talks to, " + "roughly how much battery, its price zone, and roughly how old the " + "install is. Nothing else — no id, no key, no serial, no site name, no " + @@ -199,6 +273,15 @@ "> Share daily fleet statistics" + '

Takes effect as soon as you save. No restart.

' + '
' + + '

Daily energy report

' + + '

The same switch sends one more report a day: the date, and the ' + + "day's solar generation, grid import and export, battery charge and " + + "discharge, and vehicle charge, each in whole kWh. It carries no id and " + + "nothing from the report above, so the two cannot be matched. It goes " + + "through an independent relay, so FTW Cloud reads the numbers without " + + "this box's address, and the relay sees the address without the numbers. " + + "A day with a gap in the energy record is skipped.

" + + '
' + "
" ); }, diff --git a/web/style.css b/web/style.css index 4c76dec73..a8505891f 100644 --- a/web/style.css +++ b/web/style.css @@ -1836,6 +1836,15 @@ footer { color: var(--text-dim); } +.fleet-energy-title { + margin: 18px 0 4px; + font-size: 0.9rem; +} +/* .hint carries no margin; without this the report's notes read as one slab. */ +.fleet-energy-note { + margin-top: 8px; +} + /* ---- Access settings tab ---- */ /* Who outside the house may reach it. The page reads top to bottom in the order an owner acts: what is missing, the code, the offer, what waits for From 4cfd65f7da95a333729d1946cadbf8f1f416d92e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 08:35:45 +0200 Subject: [PATCH 2/8] feat(stats): one statistics answer, and both daily reports through the relay Fredrik's change of scope, 24 October 2026: statistics are one answer on the box, and every report goes the same way. - One answer for both anonymous reports (cloudlink.SharesStatistics): part of FTW Cloud under the current terms, with no off switch; on for a box without Cloud until its owner turns it off. Nobody is asked. Stored state is read once (MigrateStatistics): a local No, a fleet report switched off in the config, or Cloud turned off on an earlier release stays off; a local yes stays yes; every other box is on by default. fleet_ping.enabled is no longer a switch. A box without Cloud no longer starts a statistics-only Cloud link. - A box that shares only by default posts nothing straight to FTW Cloud, which would see its address; it waits for the relay. - go/internal/statsrelay holds the relay and gateway pins and sends sealed reports; the fleet report goes through it once a release names a relay, the energy report always. Each report travels alone at its own time. - go/protocol/reportwire: one padded inner request for both kinds (1,536 bytes); the gateway picks the decoder by schema. - Settings: Anonymous statistics with the one switch beside both messages; the Cloud panel no longer asks local boxes. - ADR 0008 and the protocol doc record the model, the two-report rule and the open ePrivacy question. Co-Authored-By: Claude Opus 5.5 --- .changeset/anonymous-daily-energy.md | 4 +- config.example.yaml | 22 +- docs/adr/0008-ftw-cloud.md | 121 ++++++---- docs/protocol-cloud.md | 6 + go/cmd/ftw/energy_report.go | 2 +- go/cmd/ftw/energy_report_test.go | 110 ++++++++-- go/cmd/ftw/fleet_ping.go | 59 ++++- go/cmd/ftw/main.go | 1 + go/internal/api/api.go | 2 +- go/internal/api/api_fleet_ping.go | 33 ++- go/internal/api/api_fleet_ping_test.go | 15 +- go/internal/api/cloud_statistics.go | 2 +- go/internal/cloudlink/choice.go | 110 +++++++--- go/internal/cloudlink/choice_test.go | 122 +++++++---- go/internal/cloudlink/grant.go | 4 + go/internal/cloudlink/link.go | 50 +++-- go/internal/cloudlink/link_test.go | 12 +- go/internal/energyreport/energyreport.go | 157 +++---------- go/internal/energyreport/energyreport_test.go | 207 ++---------------- go/internal/fleetping/fleetping.go | 42 +++- go/internal/fleetping/fleetping_test.go | 25 +++ go/internal/statsrelay/statsrelay.go | 145 ++++++++++++ go/internal/statsrelay/statsrelay_test.go | 131 +++++++++++ .../statsrelaytest/statsrelaytest.go | 145 ++++++++++++ go/protocol/energywire/energy.go | 22 +- go/protocol/energywire/energy_test.go | 21 -- go/protocol/reportwire/report.go | 68 ++++++ go/protocol/reportwire/report_test.go | 68 ++++++ web/app-link-tab.test.mjs | 12 +- web/fleet-ping-tab.test.mjs | 68 +++--- web/project-statistics.js | 55 +++-- web/project-statistics.test.mjs | 25 +-- web/settings/tabs/app.js | 4 +- web/settings/tabs/fleet.js | 130 +++++++---- 34 files changed, 1335 insertions(+), 665 deletions(-) create mode 100644 go/internal/statsrelay/statsrelay.go create mode 100644 go/internal/statsrelay/statsrelay_test.go create mode 100644 go/internal/statsrelay/statsrelaytest/statsrelaytest.go create mode 100644 go/protocol/reportwire/report.go create mode 100644 go/protocol/reportwire/report_test.go diff --git a/.changeset/anonymous-daily-energy.md b/.changeset/anonymous-daily-energy.md index e8ecde5f7..b97cbb6a7 100644 --- a/.changeset/anonymous-daily-energy.md +++ b/.changeset/anonymous-daily-energy.md @@ -2,4 +2,6 @@ "ftw": minor --- -The box can send one anonymous energy report a day: the date and that UTC day's solar generation, grid import and export, battery charge and discharge and vehicle charge, in whole kWh. It goes through an independent Oblivious HTTP relay to FTW Cloud's gateway, so the relay never sees the report and FTW Cloud never sees the box's address. It follows the owner's existing statistics answer, skips any day with a gap in the energy record and any site a partner controls, and adds no question or setting. Settings shows the report beside the fleet report. This release names no relay yet, so it sends nothing until a later release pins one. +Statistics are one answer on the box, for two anonymous daily reports. With FTW Cloud they are part of Cloud. Without Cloud they are on until you turn them off under Settings → FTW app → Anonymous statistics, which shows both messages; nobody is asked any more. A No, a fleet report switched off, or Cloud turned off on an earlier release stays off. + +Beside the fleet report, the box can now send one energy report a day: the date and that UTC day's solar generation, grid import and export, battery charge and discharge and vehicle charge, in whole kWh, never in the same message as the fleet report and at its own time. It skips any day with a gap in the energy record and any site a partner controls. Once a release names an Oblivious HTTP relay, both reports go through it, so FTW Cloud never sees the box's address. This release names none yet: the energy report is not sent, a box that said yes sends the fleet report straight to FTW Cloud as before, and a box that shares only by default sends nothing until a relay is named. diff --git a/config.example.yaml b/config.example.yaml index 249d228ff..17cb7480d 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -220,16 +220,18 @@ state: app_link: enabled: true -# Anonymous fleet ping. Once a day, at a time that moves, the box tells the -# FTW relay six things: its FTW version, its release channel, which driver -# types it runs, roughly how much battery it has, its price zone and roughly -# how old the install is. No id, no key, no serial, no site name, no counter -# and no timestamp — nothing in the message says which box sent it. Two -# limits: those six fields still describe a household, so a rare mix may be -# the only one of its kind, and the relay sees the address during the request. -# It saves only daily totals, so the result counts reports, not unique boxes. -# Settings → FTW app → Fleet statistics shows the exact message this box would send. On by -# default; `enabled: false` turns it off without a restart. +# Anonymous daily reports. Statistics are one answer on the box: part of FTW +# Cloud, and on for a box without Cloud until its owner turns them off under +# Settings → FTW app → Anonymous statistics, which shows each message. Two +# reports, each alone and at its own time. The fleet report carries six +# things: FTW version, release channel, driver types, roughly how much +# battery, price zone and roughly how old the install is. The energy report +# carries a closed UTC day's six whole-kWh sums. No id, no key, no serial, no +# site name, no counter. Once a release names a relay, both go through it, so +# FTW Cloud never sees the address; until then the fleet report goes straight +# to `endpoint`, which sees the address, and the energy report is not sent. +# `enabled: false` from an earlier release is read once and kept as "off"; it +# is no longer a switch of its own. fleet_ping: enabled: true # endpoint: https://cloud.ftw.energy/v1/fleet diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index 15203c3b9..9f4843cfd 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -162,6 +162,9 @@ Owners who decline use the box locally without Cloud. Local-only users get a separate question, with Yes as the suggested action. Nothing starts before an active answer. No leaves local control working. +(Replaced on 24 October 2026: a box without Cloud shares the two anonymous +reports until its owner turns them off; see "Anonymous daily reports and one +statistics answer" below.) This statistics-only link publishes no phone keys, pairing codes, Noise key, read grant, live state or remote access. It reuses the signed upload path. A policy or purpose change requires a new version and a fresh answer. @@ -329,6 +332,8 @@ who declines its terms pauses Cloud on that phone and keeps its local keys. The daily anonymous report also waits for the current project choice. Its saved config switch cannot bypass a missing answer or a local No. +(Replaced on 24 October 2026; see "Anonymous daily reports and one +statistics answer" below.) A new policy rotates the stats id and starts a new reporting window. An unset wall clock starts that window only once time is known; no earlier hours are filled in. @@ -351,42 +356,70 @@ reports this readiness so native clients can explain the wait. Pausing the service policy stops collection while withdrawal and retention still run. Deploying this code does not sign the DPIA or accept a household's consent. -### Anonymous daily energy, October 2026 - -Fredrik chose this on 24 October 2026 so that stats.ftw.energy can show -energy without C8's household rows. One new outside part: an independent -Oblivious HTTP relay (RFC 9458). FTW Cloud's `ftwcloudd` is the gateway, as -one route. - -- **Sent.** Once per closed UTC day, 1 to 12 hours after it closes, at a time - drawn afresh each day: the schema, the date, and six whole-kWh sums (solar +### Anonymous daily reports and one statistics answer, October 2026 + +Fredrik decided this on 24 October 2026, so that stats.ftw.energy can show +energy without C8's household rows, and so that statistics are one thing: +"antingen delar man med sig av stats eller inte". + +**One answer.** Statistics are one answer on the box that covers both +anonymous daily reports, with no second setting: + +- With FTW Cloud they are part of Cloud, with no separate off switch, as in + the 17 October amendment. A Cloud grant under older terms sends nothing + until the owner accepts the current ones. +- Without Cloud they are on until the owner turns them off, under Settings → + FTW app → Anonymous statistics, where the box shows each message. Nobody is + asked any more; the local question and its "nothing before an answer" rule + are gone. +- Stored state is read once (`cloudlink.MigrateStatistics`). A local No, a + fleet report switched off in the config (`fleet_ping.enabled: false`), or + Cloud turned off before this release (whose screen said that stops + statistics) stays off. Every other box is on. `fleet_ping.enabled` is no + longer a switch. +- A box without Cloud no longer starts a statistics-only Cloud link. Those + already running keep working until turned off. Hourly C8 rows for local + boxes therefore have no way in; if C8 is ever activated for them, it needs + its own consent. + +**Two reports, never one message.** Each leaves at its own random time: + +- the fleet report (`go/internal/fleetping`): version, channel, drivers, + battery size, price zone, install age, as before; +- the energy report (`go/internal/energyreport`): once per closed UTC day, + 1 to 12 hours after it closes, the date and six whole-kWh sums (solar generation, grid import and export, battery charge and discharge, vehicle - charge). Nothing else: no id, version, driver, zone or size. A day with a - gap in the energy ledger is skipped, never sent in part - (`go/internal/energyreport`, `go/protocol/energywire`). -- **Who may send.** The fleet report's answer: its switch and the current - project statistics choice. Not a site a partner controls. No new question - or setting. Settings shows the report beside the fleet report. -- **Path.** The box seals the report to FTW Cloud's gateway key with HPKE - (`go/protocol/ohttp`, the standard library's `crypto/hpke`) and posts it to - the relay, which forwards it to `POST /v1/ohttp`. The inner request is - padded to one length. The key and the relay are compile-time pins - (`energyreport.GatewayKey`, `energyreport.RelayURL`). The box never asks - the service for a key, so every box on a release uses the same one and a - key swapped in at the service opens nothing. Both pins are empty until the - relay exists; until then the box sends nothing. -- **Who sees what.** The relay sees the box's address, the time and a - ciphertext of fixed length, never the report. Cloudflare's relay, if - chosen, keeps those logs for about 124 days. The gateway sees the report, - the relay's address and the time, never the box's address. -- **Kept.** Per date: the report count and the six sums, for 30 days. No row, - logged body or address per report. -- **Published.** `GET /v1/energy/public` lists closed dates only, once no - report may still name them, so two readings never differ by one report. A - date with fewer than 5 reports shows null, as `/v1/fleet/public` does. -- **Not linkable.** The energy report shares no field with the fleet report, - and the two leave at times drawn apart. Neither the relay nor the gateway - holds both a box's address and its numbers. + charge). A day with a gap in the energy ledger is skipped, never sent in + part. Never for a site a partner controls. + +With a fleet of about eight boxes, a driver set such as Sungrow, Easee, +myuplink and tesla-ha in SE4 points at one home. Energy sent beside it would +be that home's energy again. So the two share no field and never travel +together. + +**Path.** Both go through one independent Oblivious HTTP relay (RFC 9458) to +FTW Cloud's gateway, `POST /v1/ohttp` in `ftwcloudd`, as one padded inner +request each (`go/protocol/reportwire`, 1,536 bytes; the relay always sees +1,591). The gateway reads the schema, adds the report to its own day totals +(`/v1/fleet/public`, `/v1/energy/public`) and keeps nothing per report. +`go/internal/statsrelay` holds the relay URL and the gateway key as +compile-time pins. The box encrypts only to the pinned key and never asks the +service for one, so every box on a release uses the same key and a key +swapped in at the service opens nothing. Until both pins are set, the energy +report is not sent, and the fleet report posts straight to `/v1/fleet` as +before, but only from a box that said yes: FTW Cloud's terms or a local yes. +A box that shares only by default never posts straight to FTW Cloud, which +would see its address; it waits for the relay. A build with the relay sends +the fleet report to FTW Cloud only through it. `POST /v1/fleet` stays for boxes on older releases; it can go one +stable release after the first that pins the relay. A collector the household +set in `fleet_ping.endpoint` is still posted to directly. + +**Who sees what.** The relay sees the box's address, the time and a +ciphertext of fixed length, never a report. Cloudflare's relay, if chosen, +keeps those logs for about 124 days. The gateway sees each report, the +relay's address and the time, never the box's address. FTW Cloud keeps per +date only the counts and sums; a public energy date appears once no report +may still name it, four days after it ends, and needs at least 5 reports. Limits, plainly: @@ -397,11 +430,13 @@ Limits, plainly: computed from them and matched with a report. The gateway does not do this and keeps nothing to match with, but that is a rule in code, not one the cryptography enforces. -- An anonymous report cannot prove that a real box sent it. Bounds per field - and a cap per date limit junk to noise. Privacy Pass-style tokens are the - later fix; they are not built. - -Open for Fredrik, not decided here: whether reports that are anonymous to -Sourceful still need the owner's yes under the ePrivacy rule on reading -terminal equipment (Swedish LEK 9:28). The design keeps the existing yes, so -it is covered either way. C8 stays off. +- An anonymous report cannot prove that a real box sent it, and the relay + could replay one. Bounds per field and a cap per date limit junk to noise. + Privacy Pass-style tokens are the later fix; they are not built. + +**Open for Fredrik's lawyer, not decided here.** Reports that are anonymous +to Sourceful through OHTTP should be outside GDPR. Reading data from the +owner's equipment to make them may still need consent under the ePrivacy +rule (Swedish LEK, chapter 9, section 28). If it does, neither the opt-out for +boxes without Cloud nor the requirement with Cloud holds, and both need an +active yes. Fredrik chose this design; it is built as he asked. C8 stays off. diff --git a/docs/protocol-cloud.md b/docs/protocol-cloud.md index 268dbd833..bde98a520 100644 --- a/docs/protocol-cloud.md +++ b/docs/protocol-cloud.md @@ -1858,3 +1858,9 @@ codes, Noise key or browser subscriptions. It grants no remote access. Core also rejects tunnels locally in that mode and emits no live or event data. Collection stays off until the signed C8 review; the default operator setting is empty. Withdrawal and retention continue while it is paused. + +Since 24 October 2026 a box without Cloud no longer starts a statistics-only +roster: its statistics are the two anonymous daily reports, through the +Oblivious HTTP relay (ADR 0008, "Anonymous daily reports and one statistics +answer"). Statistics-only rosters already running keep working until the +owner turns statistics off. diff --git a/go/cmd/ftw/energy_report.go b/go/cmd/ftw/energy_report.go index f6e4355e9..276c2930c 100644 --- a/go/cmd/ftw/energy_report.go +++ b/go/cmd/ftw/energy_report.go @@ -40,7 +40,7 @@ func startEnergyReport(ctx context.Context, cfg *config.Config, cfgMu *sync.RWMu // controls: as for the hourly project statistics, a partner's contract has to // let such a home in first. func energyReportAllowed(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) func() bool { - allowed := projectReportsAllowed(cfg, cfgMu, st) + allowed := sharesStatistics(st) return func() bool { cfgMu.RLock() partner := cfg.Nova != nil && cfg.Nova.Enabled diff --git a/go/cmd/ftw/energy_report_test.go b/go/cmd/ftw/energy_report_test.go index 155f83f71..e8b9e9500 100644 --- a/go/cmd/ftw/energy_report_test.go +++ b/go/cmd/ftw/energy_report_test.go @@ -5,23 +5,28 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/statsrelay" + "github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest" "github.com/srcfl/ftw/go/protocol/cloudwire" ) -// The energy report has no question of its own. It sends only when the fleet -// report would, and never for a site a partner controls. -func TestTheEnergyReportTakesTheFleetReportsAnswer(t *testing.T) { +// Both reports read one answer: part of FTW Cloud, on for a box without it +// until the owner turns it off. The energy report also leaves out a site a +// partner controls. +func TestBothReportsReadTheOneStatisticsAnswer(t *testing.T) { st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) - cfg := &config.Config{} + cfg := &config.Config{FleetPing: &config.FleetPing{Enabled: true}} cfgMu := &sync.RWMutex{} - energy, fleet := energyReportAllowed(cfg, cfgMu, st), projectReportsAllowed(cfg, cfgMu, st) + energy, fleet := energyReportAllowed(cfg, cfgMu, st), sharesStatistics(st) check := func(name string, wantEnergy, wantFleet bool) { t.Helper() if energy() != wantEnergy || fleet() != wantFleet { @@ -29,23 +34,96 @@ func TestTheEnergyReportTakesTheFleetReportsAnswer(t *testing.T) { } } - check("no answer yet", false, false) - grant, _ := json.Marshal(map[string]any{"statsPolicy": cloudwire.StatsPolicy, "statsId": make([]byte, 16)}) - if err := st.SaveConfig("cloud/grant_v1", string(grant)); err != nil { - t.Fatal(err) - } - check("the owner said yes", true, true) + check("before the stored state is read", false, false) + migrateStatistics(cfg, cfgMu, st) + check("a box without Cloud, never asked", true, true) - cfg.FleetPing = &config.FleetPing{Enabled: false} - check("the daily report switched off", false, false) - cfg.FleetPing.Enabled = true + // The old switch is read once and is no longer a second setting. + cfg.FleetPing.Enabled = false + check("the retired fleet switch", true, true) cfg.Nova = &config.Nova{Enabled: true} check("a partner controls the site", false, true) cfg.Nova = nil - if err := st.SaveConfig("cloud/disabled_v1", "true"); err != nil { + if err := st.SaveConfig("statistics/share_v1", "off"); err != nil { t.Fatal(err) } - check("Cloud ended", false, false) + check("the owner turned statistics off", false, false) + + grant, _ := json.Marshal(map[string]any{"policy": cloudwire.CloudTermsPolicy, "purposes": []string{"serve", "stats"}}) + if err := st.SaveConfig("cloud/grant_v1", string(grant)); err != nil { + t.Fatal(err) + } + check("FTW Cloud under the current terms", true, true) +} + +// A box that shares only by default posts nothing straight to FTW Cloud: its +// fleet report waits for the relay. A yes, the relay or the household's own +// collector lets it go. +func TestADefaultBoxSendsNothingStraightToFTWCloud(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + migrateStatistics(&config.Config{}, &sync.RWMutex{}, st) + client, _, _ := statsrelaytest.New(t, time.Now()) + direct, _ := fleetProvider(config.DefaultFleetPingEndpoint, nil) + relayed, _ := fleetProvider(config.DefaultFleetPingEndpoint, client) + own, _ := fleetProvider("https://collector.example.test/fleet", nil) + if fleetReportAllowed(st, config.DefaultFleetPingEndpoint, direct)() { + t.Fatal("a box that never said yes posted straight to FTW Cloud") + } + if !fleetReportAllowed(st, config.DefaultFleetPingEndpoint, relayed)() || !fleetReportAllowed(st, "https://collector.example.test/fleet", own)() { + t.Fatal("the relay or the household's collector was held back") + } + if err := st.SaveConfig("statistics/share_v1", "on"); err != nil { + t.Fatal(err) + } + if !fleetReportAllowed(st, config.DefaultFleetPingEndpoint, direct)() { + t.Fatal("a yes did not send") + } +} + +// A fleet report switched off before this release stays off. +func TestASwitchedOffFleetReportStaysOff(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + cfg := &config.Config{FleetPing: &config.FleetPing{Enabled: false}} + migrateStatistics(cfg, &sync.RWMutex{}, st) + if sharesStatistics(st)() { + t.Fatal("an earlier off was turned on") + } +} + +// A build with a relay sends the fleet report to FTW Cloud only through it. +// A collector the household chose is posted to directly, and a build without +// a relay keeps the direct path. +func TestTheFleetReportTakesTheRelayToFTWCloud(t *testing.T) { + client, _, _ := statsrelaytest.New(t, time.Now()) + for name, c := range map[string]struct { + endpoint string + relay bool + want bool + }{ + "FTW Cloud with a relay": {config.DefaultFleetPingEndpoint, true, true}, + "FTW Cloud without a relay": {config.DefaultFleetPingEndpoint, false, false}, + "a household's collector": {"https://collector.example.test/fleet", true, false}, + } { + var relay *statsrelay.Client + if c.relay { + relay = client + } + p, err := fleetProvider(c.endpoint, relay) + if err != nil { + t.Fatal(err) + } + if _, through := p.(*fleetping.RelayProvider); through != c.want { + t.Fatalf("%s: through the relay %v", name, through) + } + } } diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go index 9d75d47ec..970d3e544 100644 --- a/go/cmd/ftw/fleet_ping.go +++ b/go/cmd/ftw/fleet_ping.go @@ -6,6 +6,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "sync" "time" @@ -14,6 +15,7 @@ import ( "github.com/srcfl/ftw/go/internal/drivers" "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/statsrelay" ) // The anonymous fleet ping, wired to the box. @@ -69,7 +71,7 @@ func startFleetPing( endpoint := cfg.FleetPing.Resolved() cfgMu.RUnlock() - provider, err := fleetping.NewHTTPProvider(endpoint, nil) + provider, err := fleetProvider(endpoint, statsrelay.Pinned()) if err != nil { return nil, err } @@ -77,7 +79,7 @@ func startFleetPing( installedAt := firstSeen(st, time.Now()) pinger, err := fleetping.New(fleetping.Options{ - Enabled: projectReportsAllowed(cfg, cfgMu, st), + Enabled: fleetReportAllowed(st, endpoint, provider), Facts: func() fleetping.Facts { // Drivers can be installed without restarting FTW. Read the signed // catalogue at the same time as the config so tomorrow's report and @@ -97,16 +99,53 @@ func startFleetPing( return pinger, nil } -// projectReportsAllowed is the owner's answer for both anonymous daily -// reports, the fleet report and the energy report: the saved switch and the -// project statistics choice. One function, so the two reports cannot come to -// read different answers. -func projectReportsAllowed(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) func() bool { - enabled := fleetPingEnabled(cfg, cfgMu) - return func() bool { return enabled() && cloudlink.StatisticsChoice(st) } +// fleetProvider sends the report to FTW Cloud through the relay when this +// build names one, and only then: FTW Cloud then never sees the box's +// address. A build without a relay posts straight to /v1/fleet as before. An +// endpoint the household chose is its own collector and is posted to +// directly. +func fleetProvider(endpoint string, relay *statsrelay.Client) (fleetping.Provider, error) { + if relay != nil && strings.TrimSpace(endpoint) == config.DefaultFleetPingEndpoint { + return fleetping.NewRelayProvider(relay), nil + } + return fleetping.NewHTTPProvider(endpoint, nil) +} + +// sharesStatistics is the one answer for both anonymous daily reports, the +// fleet report and the energy report: part of FTW Cloud, and on for a box +// without it until the owner turns it off. One function, so the two reports +// cannot come to read different answers. fleet_ping.enabled is no longer a +// second switch; MigrateStatistics read it once. +func sharesStatistics(st *state.Store) func() bool { + return func() bool { return st != nil && cloudlink.SharesStatistics(st) } +} + +// fleetReportAllowed is the one answer, with one more rule for the direct +// path: a box that shares only by default, never having said yes, posts +// nothing straight to FTW Cloud, which would see its address. Through the +// relay, or to a collector the household chose, the answer alone decides. +func fleetReportAllowed(st *state.Store, endpoint string, provider fleetping.Provider) func() bool { + shares := sharesStatistics(st) + _, relayed := provider.(*fleetping.RelayProvider) + direct := !relayed && strings.TrimSpace(endpoint) == config.DefaultFleetPingEndpoint + return func() bool { + return shares() && (!direct || cloudlink.SaidYesToStatistics(st)) + } +} + +// migrateStatistics turns what an earlier release stored into the one +// answer, once: a fleet report switched off in the config stays off. +func migrateStatistics(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) { + if st == nil { + return + } + if err := cloudlink.MigrateStatistics(st, !fleetPingEnabled(cfg, cfgMu)()); err != nil { + slog.Warn("could not record the statistics answer; nothing is sent until it is", "err", err) + } } -// fleetPingEnabled reads the switch, under the config lock, at each send. +// fleetPingEnabled reads the old switch under the config lock. Since the one +// statistics answer it is read once, by migrateStatistics. // // On() and not a second reading of nil. A config posted to /api/config never // passes through applyDefaults and configreload.Apply swaps it in whole, so a diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index b3738e412..88e80ef2f 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2492,6 +2492,7 @@ func main() { // The bundled drivers dir is deliberately not passed: which drivers this // build ships with is compiled in, not discovered on disk. See // fleetCatalogue for what that buys. + migrateStatistics(cfg, cfgMu, st) fleetPinger, fleetPingErr := startFleetPing( ctx, cfg, cfgMu, st, Version, config.ManagedDriversDirOverride, ) diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 166f34e47..ecc4b96a1 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -41,9 +41,9 @@ import ( "github.com/srcfl/ftw/go/internal/coverage" "github.com/srcfl/ftw/go/internal/driverrepo" "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/energyreport" "github.com/srcfl/ftw/go/internal/evcloud" "github.com/srcfl/ftw/go/internal/events" - "github.com/srcfl/ftw/go/internal/energyreport" "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/forecast" "github.com/srcfl/ftw/go/internal/ha" diff --git a/go/internal/api/api_fleet_ping.go b/go/internal/api/api_fleet_ping.go index 11d317848..66f9ea191 100644 --- a/go/internal/api/api_fleet_ping.go +++ b/go/internal/api/api_fleet_ping.go @@ -3,6 +3,7 @@ package api import ( "net/http" + "github.com/srcfl/ftw/go/internal/cloudlink" "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/energyreport" ) @@ -20,9 +21,16 @@ type fleetPingView struct { // FTWRelay lets Settings state FTW's retention rules only when the running // process is using the endpoint those rules govern. FTWRelay bool `json:"ftw_relay"` + // ThroughRelay is true when the report goes to FTW Cloud through the + // Oblivious HTTP relay, so FTW Cloud never sees the box's address. + ThroughRelay bool `json:"through_relay"` + // WaitsForRelay is a box that shares by default, never having said yes: + // its report goes to FTW Cloud only through the relay, and this build + // names none, so it waits. + WaitsForRelay bool `json:"waits_for_relay"` // Payload is the whole message, exactly as it would be posted. Payload any `json:"payload"` - // Energy is the daily energy report under the same switch, built by the + // Energy is the daily energy report under the same answer, built by the // call its sender uses. Absent on a box without one. Energy *energyreport.Preview `json:"energy,omitempty"` } @@ -35,20 +43,23 @@ func (s *Server) handleFleetPing(w http.ResponseWriter, r *http.Request) { return } - // The switch is live, but the endpoint is fixed when the pinger starts. - // Report that active endpoint rather than a newly saved value that needs a - // restart before it can be used. - s.deps.CfgMu.RLock() - enabled := s.deps.Cfg.FleetPing.On() - s.deps.CfgMu.RUnlock() + // Enabled is whether the next report goes, read the way the sender reads + // it: the box's one statistics answer, and no direct post to FTW Cloud + // from a box that never said yes. The endpoint is fixed when the pinger + // starts. Report that active endpoint rather than a newly saved value + // that needs a restart before it can be used. + enabled := s.deps.FleetPing.Enabled() endpoint := s.deps.FleetPing.Endpoint() view := fleetPingView{ - Enabled: enabled, - Endpoint: endpoint, - FTWRelay: endpoint == config.DefaultFleetPingEndpoint, - Payload: s.deps.FleetPing.Payload(), + Enabled: enabled, + Endpoint: endpoint, + FTWRelay: endpoint == config.DefaultFleetPingEndpoint || s.deps.FleetPing.ThroughRelay(), + ThroughRelay: s.deps.FleetPing.ThroughRelay(), + Payload: s.deps.FleetPing.Payload(), } + view.WaitsForRelay = !enabled && !view.ThroughRelay && endpoint == config.DefaultFleetPingEndpoint && + s.deps.State != nil && cloudlink.SharesStatistics(s.deps.State) if s.deps.EnergyReport != nil { p := s.deps.EnergyReport.Preview(r.Context()) view.Energy = &p diff --git a/go/internal/api/api_fleet_ping_test.go b/go/internal/api/api_fleet_ping_test.go index 3a30a8826..ed543683d 100644 --- a/go/internal/api/api_fleet_ping_test.go +++ b/go/internal/api/api_fleet_ping_test.go @@ -5,10 +5,12 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "path/filepath" "sync" "testing" "time" + "github.com/srcfl/ftw/go/internal/cloudlink" "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/energyreport" "github.com/srcfl/ftw/go/internal/fleetping" @@ -26,7 +28,13 @@ func fleetPingServer(t *testing.T, cfg *config.Config) *Server { func fleetPingServerAt(t *testing.T, cfg *config.Config, endpoint string) *Server { t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) pinger, err := fleetping.New(fleetping.Options{ + Enabled: func() bool { return cloudlink.SharesStatistics(st) }, Provider: silentProvider(endpoint), Facts: func() fleetping.Facts { return fleetping.FactsFromConfig(cfg, "v1.4.0", @@ -37,7 +45,12 @@ func fleetPingServerAt(t *testing.T, cfg *config.Config, endpoint string) *Serve if err != nil { t.Fatalf("fleetping.New: %v", err) } - return New(&Deps{CfgMu: &sync.RWMutex{}, Cfg: cfg, FleetPing: pinger}) + // The box's one statistics answer, written from this config the way + // Core writes it once at start. + if err := cloudlink.MigrateStatistics(st, !cfg.FleetPing.On()); err != nil { + t.Fatal(err) + } + return New(&Deps{CfgMu: &sync.RWMutex{}, Cfg: cfg, FleetPing: pinger, State: st}) } func TestFleetPingShowsTheRealPayload(t *testing.T) { diff --git a/go/internal/api/cloud_statistics.go b/go/internal/api/cloud_statistics.go index 5f47fea75..3769f2d63 100644 --- a/go/internal/api/cloud_statistics.go +++ b/go/internal/api/cloud_statistics.go @@ -140,6 +140,6 @@ func (s *Server) handleCloudStatsPolicy(w http.ResponseWriter, r *http.Request) "retention": "48 hours. Leaving Cloud or stopping local sharing deletes unpublished household rows; published group totals remain.", "publication_minimum_homes": cloudwire.StatsMinimumHomes, "publication_max_home_share": 0.3, - "controller": "sourceful", "local_default": "ask_yes", "local_optional": true, "cloud_required": true, + "controller": "sourceful", "local_default": "on", "local_optional": true, "cloud_required": true, }) } diff --git a/go/internal/cloudlink/choice.go b/go/internal/cloudlink/choice.go index 776a265a7..07cc677d6 100644 --- a/go/internal/cloudlink/choice.go +++ b/go/internal/cloudlink/choice.go @@ -6,7 +6,7 @@ import ( "github.com/srcfl/ftw/go/protocol/cloudwire" ) -var ErrCloudNeedsStatistics = errors.New("cloudlink: statistics are part of FTW Cloud; turn FTW Cloud off to stop contributing") +var ErrCloudNeedsStatistics = errors.New("cloudlink: statistics are part of FTW Cloud; turn FTW Cloud off, then statistics, to stop contributing") // includeStatistics records the exact scope the owner read. Service approval // still gates every report. The caller holds mu. @@ -34,36 +34,6 @@ func (l *Link) enableLinkChoice() error { return nil } -// startStatisticsOnly reuses the signed uplink, without publishing any phone, -// pairing code, Noise key, live reading or grant for remote access. -func (l *Link) startStatisticsOnly(req StatsRequest) error { - if err := l.linkLocked(); err != nil { - return err - } - if l.link.Ending { - return ErrLinkEnding - } - id, err := randomBytes(cloudwire.GrantIDBytes) - if err != nil { - return err - } - key, err := randomBytes(cloudwire.GrantKeyBytes) - if err != nil { - return err - } - g := grantRecord{ID: id, GrantKey: key, KeyFp: l.link.KeyFp, Policy: req.Policy, By: req.By, Controller: cloudwire.ControllerSourceful, Scopes: []string{}, Purposes: []string{protocol.CloudPurposeStats}} - if l.clockSynced() { - g.AtMs = l.opts.Now().UnixMilli() - } - if err := l.includeStatistics(&g, true); err != nil { - return err - } - if err := l.enableLinkChoice(); err != nil { - return err - } - return l.storeLocked(g, "local statistics accepted") -} - // End stops all Cloud functions, purges the home and remembers the choice so // a paired phone or a restart cannot silently turn the link back on. func (l *Link) End(by string) error { @@ -103,17 +73,87 @@ func (l *Link) End(by string) error { return nil } -// StatisticsChoice is the saved answer for both project reports. The daily -// report must not turn itself on before the owner answers the new question. -func StatisticsChoice(kv KV) bool { +// SharesStatistics is the one statistics answer, for both anonymous daily +// reports: the fleet report and the energy report (Fredrik, 24 October 2026). +// With FTW Cloud they are part of Cloud, with no separate off switch. A Cloud +// grant under older terms sends nothing until the owner accepts the current +// ones. Without Cloud they are on unless the owner turned them off. +func SharesStatistics(kv KV) bool { if kv == nil { return false } + g, err := loadJSON[grantRecord](kv, grantKey) + if err != nil { + return false + } + // Cloud turned off is off at once, before the grant record says so. if disabled, _ := kv.LoadConfig(disabledKey); disabled == "true" { + g = nil + } + share, _ := kv.LoadConfig(statsShareKey) + return sharesStatistics(g, share) +} + +func sharesStatistics(g *grantRecord, share string) bool { + if g != nil && !g.Revoked && !g.statisticsOnly() { + return g.Policy == cloudwire.CloudTermsPolicy + } + return share == "on" || share == "default" +} + +// SaidYesToStatistics reports an active yes: FTW Cloud under the current +// terms, or a local yes. A box that shares only by default has not given one, +// so its reports reach FTW Cloud only through the relay, which hides its +// address; nothing goes straight to FTW Cloud from it. +func SaidYesToStatistics(kv KV) bool { + if kv == nil { return false } g, err := loadJSON[grantRecord](kv, grantKey) - return err == nil && g != nil && !g.Revoked && g.StatsPolicy == cloudwire.StatsPolicy && len(g.StatsID) == 16 + if err != nil { + return false + } + if disabled, _ := kv.LoadConfig(disabledKey); disabled == "true" { + g = nil + } + if g != nil && !g.Revoked && !g.statisticsOnly() { + return g.Policy == cloudwire.CloudTermsPolicy + } + share, _ := kv.LoadConfig(statsShareKey) + return share == "on" +} + +// cloudRequiresStatistics reports whether the grant is Cloud under the +// current terms, where statistics have no separate off switch. +func cloudRequiresStatistics(g *grantRecord) bool { + return g != nil && !g.Revoked && g.Policy == cloudwire.CloudTermsPolicy +} + +// MigrateStatistics writes the one answer once, from what an earlier release +// stored. A local No, a fleet report switched off, or FTW Cloud turned off +// (whose screen said that stops statistics) stays off. A local yes stays yes. +// Every other box is on by default: a box without Cloud shares unless its +// owner turns it off. Until this has run, SharesStatistics reads a box +// without Cloud as off. +func MigrateStatistics(kv KV, fleetReportOff bool) error { + if v, ok := kv.LoadConfig(statsShareKey); ok && (v == "on" || v == "off" || v == "default") { + return nil + } + g, err := loadJSON[grantRecord](kv, grantKey) + if err != nil { + return err + } + decision, _ := kv.LoadConfig(statsDecisionKey) + disabled, _ := kv.LoadConfig(disabledKey) + answeredYes := g != nil && !g.Revoked && g.StatsPolicy == cloudwire.StatsPolicy && len(g.StatsID) == 16 + share := "default" + switch { + case fleetReportOff || disabled == "true" || (decision != "" && !answeredYes): + share = "off" + case answeredYes && g.statisticsOnly(): + share = "on" + } + return kv.SaveConfig(statsShareKey, share) } // An unset clock cannot place a choice in UTC. Start its upload window only diff --git a/go/internal/cloudlink/choice_test.go b/go/internal/cloudlink/choice_test.go index be516aa8b..805383566 100644 --- a/go/internal/cloudlink/choice_test.go +++ b/go/internal/cloudlink/choice_test.go @@ -24,44 +24,34 @@ func TestCloudTermsIncludeStatisticsAndCannotOptOutSeparately(t *testing.T) { if err := h.link.StopStats("owner"); !errors.Is(err, ErrCloudNeedsStatistics) { t.Fatal("Cloud could opt out independently", err) } - if !h.link.Status().CloudTermsAccepted || !h.link.Status().StatisticsActive { + if s := h.link.Status(); !s.CloudTermsAccepted || !s.StatisticsActive || !s.Statistics || !s.StatisticsRequired { t.Fatal("new terms not active") } } -func TestLocalStatisticsPublishesNoRemoteAccess(t *testing.T) { +// Without Cloud, statistics are an answer on the box. Turning them on or off +// links nothing to Cloud and publishes no remote access. +func TestLocalStatisticsLinkNothingToCloud(t *testing.T) { svc := newFakeService(t) h := newHarness(t, svc) + if s := h.link.Status(); s.StatisticsDecisionRequired || s.Statistics || s.StatisticsRequired { + t.Fatal("a box asked, or shared before its answer was recorded", s) + } if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy, By: "local owner"}); err != nil { t.Fatal(err) } - r := h.nextRoster(t) - if len(r.Devices) != 0 || len(r.Pending) != 0 || len(r.Noise) != 0 || len(r.WebPush) != 0 || len(r.Grant.Scopes) != 0 || len(r.Grant.Purposes) != 1 || r.Grant.Purposes[0] != protocol.CloudPurposeStats { - t.Fatal("statistics-only exposed Cloud access", r) + s := h.link.Status() + if !s.Statistics || s.StatisticsRequired || s.Link != "off" || s.StatisticsOnly || !SharesStatistics(h.kv) { + t.Fatal("local statistics are not a local answer", s) } - if h.link.TakesTun() || h.link.Home() != "" || h.link.SessionInfo() != nil || h.link.current().copyOn() { + if h.link.Home() != "" || h.link.SessionInfo() != nil || h.link.current().copyOn() || h.link.current().g != nil { t.Fatal("local statistics enabled Cloud functions") } - if err := h.link.SendTun(cloudwire.Tun{}, false); !errors.Is(err, ErrNoUplink) { - t.Fatal("statistics-only queued a tunnel", err) - } - if !h.link.Status().StatisticsOnly || h.link.Status().State != "statistics" { - t.Fatal("local statistics labelled Cloud") - } -} - -func TestDecliningLocalStatisticsRecordsTheAnswerWithoutCreatingALink(t *testing.T) { - svc := newFakeService(t) - h := newHarness(t, svc) - if !h.link.Status().StatisticsDecisionRequired { - t.Fatal("an unanswered choice looked accepted") - } if err := h.link.StopStats("local owner"); err != nil { t.Fatal(err) } - s := h.link.Status() - if s.StatisticsDecisionRequired || s.Link != "off" || s.StatisticsActive { - t.Fatal("decline did not stay local", s) + if s := h.link.Status(); s.Statistics || s.Link != "off" || s.StatisticsActive || SharesStatistics(h.kv) { + t.Fatal("off did not stay off and local", s) } } @@ -83,31 +73,83 @@ func TestEndingCloudPreventsAutomaticRelinkFromPairedPhones(t *testing.T) { } } -func TestDailyReportRequiresCurrentAnswerAndHonoursDurableOff(t *testing.T) { - kv := &memKV{} - if StatisticsChoice(kv) { - t.Fatal("unanswered box may send") +// Every earlier No stays no; every other box shares, since a box without +// Cloud shares until its owner turns it off. The answer is written once. +func TestMigrationKeepsEveryEarlierNo(t *testing.T) { + yes := grantRecord{StatsPolicy: cloudwire.StatsPolicy, StatsID: make([]byte, 16), Purposes: []string{protocol.CloudPurposeStats}} + for name, c := range map[string]struct { + kv map[string]string + grant *grantRecord + fleetOff bool + want bool + }{ + "never asked": {want: true}, + "local yes": {grant: &yes, want: true}, + "local no": {kv: map[string]string{statsDecisionKey: cloudwire.StatsPolicy}, want: false}, + "no to an older policy": {kv: map[string]string{statsDecisionKey: "ftw-project-stats-2026-10-v1"}, want: false}, + "Cloud turned off": {kv: map[string]string{disabledKey: "true"}, want: false}, + "fleet report off": {fleetOff: true, want: false}, + "yes, fleet report off": {grant: &yes, fleetOff: true, want: false}, + "already answered: on": {kv: map[string]string{statsShareKey: "on", disabledKey: "true"}, want: true}, + "already on by default": {kv: map[string]string{statsShareKey: "default"}, want: true}, + "already answered: off": {kv: map[string]string{statsShareKey: "off"}, want: false}, + } { + kv := &memKV{m: map[string]string{}} + for k, v := range c.kv { + kv.m[k] = v + } + if c.grant != nil { + if err := saveJSON(kv, grantKey, *c.grant); err != nil { + t.Fatal(err) + } + } + if SharesStatistics(kv) && c.kv[statsShareKey] == "" { + t.Fatalf("%s: shared before the answer was written", name) + } + if err := MigrateStatistics(kv, c.fleetOff); err != nil { + t.Fatal(err) + } + if got := SharesStatistics(kv); got != c.want { + t.Fatalf("%s: shares %v, want %v", name, got, c.want) + } + // Only a yes that was given counts as one. + if said := SaidYesToStatistics(kv); said != (c.want && (c.grant != nil || c.kv[statsShareKey] == "on")) { + t.Fatalf("%s: said yes %v", name, said) + } + // Written once: a later start reads the answer, not the old flags. + if err := MigrateStatistics(kv, !c.fleetOff); err != nil || SharesStatistics(kv) != c.want { + t.Fatalf("%s: a second start changed the answer", name) + } } - g := grantRecord{StatsPolicy: "ftw-project-stats-2026-10-v1", StatsID: make([]byte, 16)} - if err := saveJSON(kv, grantKey, g); err != nil { +} + +// With Cloud under the current terms, statistics are part of Cloud whatever +// the local answer; under older terms nothing goes until they are accepted. +func TestCloudDecidesStatisticsWhileItIsOn(t *testing.T) { + kv := &memKV{m: map[string]string{statsShareKey: "off"}} + cloud := grantRecord{Policy: cloudwire.CloudTermsPolicy, Purposes: []string{protocol.CloudPurposeServe, protocol.CloudPurposeStats}} + if err := saveJSON(kv, grantKey, cloud); err != nil { t.Fatal(err) } - if StatisticsChoice(kv) { - t.Fatal("old choice may send new report") + if !SharesStatistics(kv) { + t.Fatal("Cloud under the current terms did not share") } - g.StatsPolicy = cloudwire.StatsPolicy - if err := saveJSON(kv, grantKey, g); err != nil { + older := cloud + older.Policy = "ftw-cloud-2026-10-v2" + kv.m[statsShareKey] = "on" + if err := saveJSON(kv, grantKey, older); err != nil { t.Fatal(err) } - if !StatisticsChoice(kv) { - t.Fatal("current answer may not send") + if SharesStatistics(kv) { + t.Fatal("Cloud under older terms shared") } - // This flag is saved first, even if writing a later purge record fails. - if err := kv.SaveConfig(disabledKey, "true"); err != nil { + revoked := cloud + revoked.Revoked = true + if err := saveJSON(kv, grantKey, revoked); err != nil { t.Fatal(err) } - if StatisticsChoice(kv) { - t.Fatal("durable Cloud off did not stop daily report") + if !SharesStatistics(kv) { + t.Fatal("a box leaving Cloud lost its own answer") } } @@ -166,7 +208,7 @@ func TestWholeCloudOffStopsImmediatelyWhenLaterStorageFails(t *testing.T) { if err := h.link.End("owner"); err == nil { t.Fatal("storage failure was not returned") } - if h.link.TakesTun() || h.link.Home() != "" || h.link.SessionInfo() != nil || h.link.current().copyOn() || StatisticsChoice(h.kv) { + if h.link.TakesTun() || h.link.Home() != "" || h.link.SessionInfo() != nil || h.link.current().copyOn() || SharesStatistics(h.kv) { t.Fatal("a failed follow-up write left Cloud sharing active") } if disabled, _ := h.kv.LoadConfig(disabledKey); disabled != "true" { diff --git a/go/internal/cloudlink/grant.go b/go/internal/cloudlink/grant.go index 625ad0c8b..5a26160c8 100644 --- a/go/internal/cloudlink/grant.go +++ b/go/internal/cloudlink/grant.go @@ -28,6 +28,10 @@ const ( purgedKey = "cloud/purged_v1" disabledKey = "cloud/disabled_v1" statsDecisionKey = "cloud/statistics_decision_v1" + // statsShareKey is the one statistics answer of a box without FTW Cloud: + // "on" (the owner said yes), "default" (on, never answered) or "off" + // (MigrateStatistics, SharesStatistics). + statsShareKey = "statistics/share_v1" ) // linkRecord is the box's link to FTW Cloud: which cloud.key it holds. The diff --git a/go/internal/cloudlink/link.go b/go/internal/cloudlink/link.go index 5d5cf0c03..300588a2e 100644 --- a/go/internal/cloudlink/link.go +++ b/go/internal/cloudlink/link.go @@ -772,13 +772,12 @@ type StatsRequest struct { By string `json:"-"` } -// Stats adds the stats purpose to a grant already on. The first time, it -// draws the statsId that names this box's rows at the service; later calls -// keep it. It sends no statistics itself. +// Stats turns statistics on. For a box without FTW Cloud that is the one +// answer on the box, and nothing links to Cloud. Under a grant already on it +// also adds the stats purpose; the first time, it draws the statsId that +// names this box's rows at the service, and later calls keep it. It sends no +// statistics itself. func (l *Link) Stats(req StatsRequest) error { - if len(l.opts.ServiceKeys) == 0 { - return ErrNoService - } if strings.TrimSpace(req.Policy) == "" { return fmt.Errorf("%w: name the policy the owner accepted", ErrGrantRequest) } @@ -789,9 +788,18 @@ func (l *Link) Stats(req StatsRequest) error { if req.Policy != cloudwire.StatsPolicy { return ErrNotGranted } - return l.startStatisticsOnly(req) + if err := l.opts.KV.SaveConfig(statsShareKey, "on"); err != nil { + return err + } + l.log.Info("audit: statistics on", "by", req.By) + return nil case l.grant.Revoked: return ErrPurgePending + case len(l.opts.ServiceKeys) == 0: + return ErrNoService + } + if err := l.opts.KV.SaveConfig(statsShareKey, "on"); err != nil { + return err } g := *l.grant if len(g.StatsID) == 0 || g.StatsPolicy != req.Policy { @@ -812,16 +820,21 @@ func (l *Link) Stats(req StatsRequest) error { return l.storeLocked(g, "statistics on") } -// StopStats removes the stats purpose and forgets the statsId. Turning it on -// again draws a new one. +// StopStats turns statistics off for a box without FTW Cloud, removes the +// stats purpose and forgets the statsId. Turning it on again draws a new one. +// With Cloud under the current terms statistics are part of Cloud. func (l *Link) StopStats(by string) error { l.mu.Lock() - if l.grant != nil && !l.grant.Revoked && l.grant.Policy == cloudwire.CloudTermsPolicy { + if cloudRequiresStatistics(l.grant) { l.mu.Unlock() return ErrCloudNeedsStatistics } only := l.grant != nil && l.grant.statisticsOnly() l.mu.Unlock() + if err := l.opts.KV.SaveConfig(statsShareKey, "off"); err != nil { + return err + } + l.log.Info("audit: statistics off", "by", by) if only { if err := l.End(by); err != nil { return err @@ -937,6 +950,10 @@ type Status struct { CloudTermsAccepted bool `json:"cloudTermsAccepted"` StatisticsOnly bool `json:"statisticsOnly"` StatisticsDecisionRequired bool `json:"statisticsDecisionRequired"` + // Statistics is the one answer for both anonymous daily reports, and + // StatisticsRequired says it is part of FTW Cloud, with no off switch. + Statistics bool `json:"statistics"` + StatisticsRequired bool `json:"statisticsRequired"` // State is the readable copy's: off, on or revoking. State string `json:"state"` StatisticsPolicy string `json:"statisticsPolicy"` @@ -995,11 +1012,16 @@ func (l *Link) Status() Status { TunDropped: l.tunDropped, } s.CloudPolicy = cloudwire.CloudTermsPolicy - decision, _ := l.opts.KV.LoadConfig(statsDecisionKey) - s.StatisticsDecisionRequired = decision != cloudwire.StatsPolicy - if l.grant != nil && !l.grant.Revoked && l.grant.StatsPolicy == cloudwire.StatsPolicy && len(l.grant.StatsID) == 16 { - s.StatisticsDecisionRequired = false + // Nobody is asked any more: statistics are part of Cloud, and on for a + // box without it until its owner turns them off. The field stays false + // for clients that still read it. + share, _ := l.opts.KV.LoadConfig(statsShareKey) + g := l.grant + if l.disabled { + g = nil } + s.Statistics = sharesStatistics(g, share) + s.StatisticsRequired = cloudRequiresStatistics(g) s.StatisticsOnly = l.grant.statisticsOnly() s.CloudTermsAccepted = l.grant != nil && !l.grant.Revoked && l.grant.Policy == cloudwire.CloudTermsPolicy s.StatisticsPolicy = cloudwire.StatsPolicy diff --git a/go/internal/cloudlink/link_test.go b/go/internal/cloudlink/link_test.go index 1fc3d2947..bb78b8b38 100644 --- a/go/internal/cloudlink/link_test.go +++ b/go/internal/cloudlink/link_test.go @@ -1253,13 +1253,19 @@ func TestNoStatisticsWhileTheCopyIsBeingDeleted(t *testing.T) { } } -func TestABuildWithoutAServiceRefusesStatistics(t *testing.T) { - link, err := New(Options{Site: &fakeSite{}, Roster: fakeRoster{}, KV: &memKV{}, +// Without Cloud the statistics answer is the box's own, so a build with no +// service still records it. It still names the policy the owner read. +func TestABuildWithoutAServiceStillRecordsTheLocalAnswer(t *testing.T) { + kv := &memKV{} + link, err := New(Options{Site: &fakeSite{}, Roster: fakeRoster{}, KV: kv, KeyPath: filepath.Join(t.TempDir(), "cloud.key"), ServiceKeys: []*ecdsa.PublicKey{}}) if err != nil { t.Fatal(err) } - if err := link.Stats(StatsRequest{Policy: "p"}); !errors.Is(err, ErrNoService) { + if err := link.Stats(StatsRequest{Policy: "p"}); !errors.Is(err, ErrNotGranted) { + t.Fatalf("an unknown policy: %v", err) + } + if err := link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy}); err != nil || !SharesStatistics(kv) { t.Fatalf("statistics: %v", err) } } diff --git a/go/internal/energyreport/energyreport.go b/go/internal/energyreport/energyreport.go index 100708ac2..75ce7451b 100644 --- a/go/internal/energyreport/energyreport.go +++ b/go/internal/energyreport/energyreport.go @@ -3,8 +3,8 @@ // can tell which home sent it. // // The report is six whole-kWh sums of one closed UTC day and its date -// (package energywire). The box seals it to FTW Cloud's gateway key, pinned -// in this release, and posts it to the relay: +// (package energywire). Package statsrelay seals it to FTW Cloud's gateway +// key, pinned in this release, and posts it to the relay: // // - the relay sees the box's address, the time and a ciphertext of fixed // length, never the report; @@ -12,72 +12,28 @@ // It adds the sums to that day's totals and keeps nothing of the report. // // Two things keep the report from being joined to the daily fleet report, -// which goes straight to FTW Cloud from the box's own address: it carries no -// field the fleet report carries, and it is sent at its own time, drawn -// afresh for each day from 1 to 12 hours after the day closes. +// which carries the box's drivers, zone and size: it carries no field the +// fleet report carries, and it is sent alone at its own time, drawn afresh +// for each day from 1 to 12 hours after the day closes. With a fleet of a +// few boxes those fields can point at one home, so energy sent beside them +// would be that home's again. // -// It runs only when the owner's answer to the project statistics question -// allows the fleet report, and never for a site a partner controls. It adds -// no question or setting of its own. +// It runs only while the box shares statistics (cloudlink.SharesStatistics), +// the one answer that also covers the fleet report, and never for a site a +// partner controls. It adds no question or setting of its own. package energyreport import ( - "bytes" "context" - "encoding/base64" "encoding/json" "errors" - "fmt" - "io" "log/slog" - "mime" - "net/http" - "net/url" "time" + "github.com/srcfl/ftw/go/internal/statsrelay" "github.com/srcfl/ftw/go/protocol/energywire" - "github.com/srcfl/ftw/go/protocol/ohttp" ) -// Compile-time settings, as in cloudlink: a relay a household could point -// elsewhere is a relay a support call could point at an attacker's. A local -// test build may set them with -ldflags -X. -var ( - // RelayURL is the OHTTP relay the report goes through, run by someone - // other than Sourceful. Empty until that relay exists: the box then - // sends nothing. - RelayURL = "" - // GatewayKey is FTW Cloud's gateway key configuration (RFC 9458, - // section 3), base64url, as - // https://cloud.ftw.energy/.well-known/ohttp-gateway serves it. The box - // encrypts only to this key and never asks the service for one, so every - // box on a release uses the same key and a key swapped in at the service - // reads nothing. Empty means no gateway key exists for this build. - GatewayKey = "" -) - -// Pinned returns this build's relay and gateway key. ok is false unless both -// are set and the key offers the suite package ohttp speaks. -func Pinned() (relay string, key ohttp.KeyConfig, ok bool) { - return pinned(RelayURL, GatewayKey) -} - -func pinned(relay, key string) (string, ohttp.KeyConfig, bool) { - raw, err := base64.RawURLEncoding.DecodeString(key) - if err != nil || relay == "" || key == "" { - return "", ohttp.KeyConfig{}, false - } - cfg, err := ohttp.ParseKeyConfig(raw) - if err != nil { - return "", ohttp.KeyConfig{}, false - } - u, err := url.Parse(relay) - if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == energywire.Authority { - return "", ohttp.KeyConfig{}, false - } - return relay, cfg, true -} - const ( // The send window after a day closes: late enough that the day's last // readings are in the ledger, early enough that the gateway still takes @@ -85,9 +41,6 @@ const ( firstSend = time.Hour lastSend = 12 * time.Hour - sendTimeout = 30 * time.Second - maxResponseBytes = 1 << 10 - // lastDayKey records the last date this box sent, on the box only, so a // restart does not send a day twice. lastDayKey = "energy_report.last_date" @@ -106,11 +59,8 @@ type Options struct { Ledger Ledger KV KV - // Relay and Key default to Pinned. Tests set them to a fake relay. - Relay string - Key *ohttp.KeyConfig - - Client *http.Client + // Relay defaults to statsrelay.Pinned. Tests set a fake relay. + Relay *statsrelay.Client Logger *slog.Logger // Injectable seams. Production leaves them nil. @@ -124,10 +74,7 @@ type Sender struct { enabled func() bool ledger Ledger kv KV - relay string - key ohttp.KeyConfig - ready bool - client *http.Client + relay *statsrelay.Client log *slog.Logger now func() time.Time random func() float64 @@ -138,24 +85,14 @@ func New(opts Options) (*Sender, error) { if opts.Ledger == nil { return nil, errors.New("energyreport: Ledger is required") } - s := &Sender{enabled: opts.Enabled, ledger: opts.Ledger, kv: opts.KV, client: opts.Client, log: opts.Logger, + s := &Sender{enabled: opts.Enabled, ledger: opts.Ledger, kv: opts.KV, relay: opts.Relay, log: opts.Logger, now: opts.Now, random: opts.Random, synced: opts.ClockSynced} - if opts.Key != nil { - s.relay, s.key, s.ready = opts.Relay, *opts.Key, opts.Relay != "" - } else { - s.relay, s.key, s.ready = Pinned() + if s.relay == nil { + s.relay = statsrelay.Pinned() } if s.enabled == nil { s.enabled = func() bool { return false } } - if s.client == nil { - s.client = &http.Client{Timeout: sendTimeout} - } else { - c := *s.client - s.client = &c - } - // A redirect would resend the sealed report to an address nobody chose. - s.client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } if s.log == nil { s.log = slog.Default() } @@ -188,7 +125,10 @@ type Preview struct { // Preview builds yesterday's report with the same call the sender uses. func (s *Sender) Preview(ctx context.Context) Preview { day := s.now().UTC().Truncate(24*time.Hour).AddDate(0, 0, -1) - p := Preview{Relay: s.relay, Sending: s.ready && s.enabled(), Day: day.Format(time.DateOnly)} + p := Preview{Day: day.Format(time.DateOnly)} + if s.relay != nil { + p.Relay, p.Sending = s.relay.Relay(), s.enabled() + } if r, ok, err := Day(ctx, s.ledger, day); err == nil && ok { p.Payload = &r } @@ -198,7 +138,7 @@ func (s *Sender) Preview(ctx context.Context) Preview { // Run sends once a day until ctx ends. Without a relay and a gateway key it // returns at once and sends nothing. func (s *Sender) Run(ctx context.Context) { - if !s.ready { + if s.relay == nil { return } var p plan @@ -267,7 +207,7 @@ func (s *Sender) sent(day time.Time) bool { // send makes one attempt for one day and forgets about it. There is no // retry: a lost report is one fewer in a count that says it counts reports. func (s *Sender) send(ctx context.Context, day time.Time) { - if !s.ready || !s.synced() || !s.enabled() || s.sent(day) { + if s.relay == nil || !s.synced() || !s.enabled() || s.sent(day) { return } r, ok, err := Day(ctx, s.ledger, day) @@ -286,56 +226,11 @@ func (s *Sender) send(ctx context.Context, day time.Time) { return } } - sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) - defer cancel() - if err := s.post(sendCtx, r); err != nil { - s.log.Debug("energy report did not go through", "err", err) - } -} - -// post seals one report and reads the gateway's answer. -func (s *Sender) post(ctx context.Context, r energywire.Report) error { body, err := json.Marshal(r) if err != nil { - return err - } - inner := ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, - Path: energywire.Path, Content: body}.Marshal(energywire.RequestSize) - sealed, reader, err := ohttp.Encapsulate(s.key, inner) - if err != nil { - return err - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.relay, bytes.NewReader(sealed)) - if err != nil { - return err - } - req.Header.Set("Content-Type", ohttp.RequestType) - // Pinned, as for the fleet report: the relay sees it, and Go's default - // names the runtime's version. - req.Header.Set("User-Agent", "ftw-energy-report") - resp, err := s.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) - if err != nil { - return err - } - media, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) - if resp.StatusCode != http.StatusOK || media != ohttp.ResponseType || len(data) > maxResponseBytes { - return fmt.Errorf("energy report: relay answered %d %q", resp.StatusCode, media) - } - msg, err := reader.Open(data) - if err != nil { - return err - } - answer, err := ohttp.ParseResponse(msg) - if err != nil { - return err + return } - if answer.Status != http.StatusNoContent { - return fmt.Errorf("energy report: gateway answered %d", answer.Status) + if err := s.relay.Send(ctx, body); err != nil { + s.log.Debug("energy report did not go through", "err", err) } - return nil } diff --git a/go/internal/energyreport/energyreport_test.go b/go/internal/energyreport/energyreport_test.go index 6aebaa1cd..b6b19863e 100644 --- a/go/internal/energyreport/energyreport_test.go +++ b/go/internal/energyreport/energyreport_test.go @@ -1,57 +1,16 @@ package energyreport import ( - "bytes" "context" - "crypto/ecdh" - "encoding/base64" - "io" "net/http" - "net/http/httptest" "sync" "testing" "time" "github.com/srcfl/ftw/go/internal/state" - "github.com/srcfl/ftw/go/protocol/energywire" - "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest" ) -// The release names no relay and no gateway key yet, so it sends nothing. -// Once both are pinned they must be a usable pair. -func TestTheReleasePinsBothOrNeither(t *testing.T) { - _, _, ok := Pinned() - switch { - case RelayURL == "" && GatewayKey == "": - if ok { - t.Fatal("nothing pinned, yet ready") - } - case !ok: - t.Fatalf("the pinned relay %q and gateway key do not make a usable pair", RelayURL) - } -} - -func TestAPinIsRefusedUnlessItIsARelayAndAKey(t *testing.T) { - key := base64.RawURLEncoding.EncodeToString(testGateway(t).Configs()[0].Marshal()) - if _, _, ok := pinned("https://relay.example.net/ftw", key); !ok { - t.Fatal("a good pair was refused") - } - for name, p := range map[string][2]string{ - "no relay": {"", key}, - "no key": {"https://relay.example.net/ftw", ""}, - "plain http": {"http://relay.example.net/ftw", key}, - "FTW Cloud itself": {"https://cloud.ftw.energy/v1/ohttp", key}, - "query": {"https://relay.example.net/ftw?box=1", key}, - "credentials": {"https://a:b@relay.example.net/ftw", key}, - "bad key": {"https://relay.example.net/ftw", "AAAA"}, - "padded base64": {"https://relay.example.net/ftw", key + "="}, - } { - if _, _, ok := pinned(p[0], p[1]); ok { - t.Fatalf("%s was accepted", name) - } - } -} - func TestTheSendTimeIsDrawnInTheWindowAfterTheDayCloses(t *testing.T) { closed := day.AddDate(0, 0, 1) for name, c := range map[string]struct { @@ -85,48 +44,30 @@ func TestTheSendTimeIsDrawnInTheWindowAfterTheDayCloses(t *testing.T) { } } -// Box to relay to gateway: the relay sees a fixed-length ciphertext and no -// report, the gateway reads the report and nothing that names the box, and -// the box hears 204. +// Box to relay to gateway: the gateway reads the day's report, and a restart +// does not send the day again. func TestABoxReportReachesTheGatewayThroughTheRelay(t *testing.T) { - g := testGateway(t) - gw := newFakeGateway(t, g) - relay := newFakeRelay(t, gw.URL) + client, gw, relay := statsrelaytest.New(t, day.Add(30*time.Hour)) kv := memKV{} - cfg := g.Configs()[0] - s := newSender(t, Options{Ledger: whole(t), KV: kv, Relay: relay.server.URL, Key: &cfg, Enabled: func() bool { return true }, + s := newSender(t, Options{Ledger: whole(t), KV: kv, Relay: client, Enabled: func() bool { return true }, Now: func() time.Time { return day.Add(30 * time.Hour) }}) s.send(context.Background(), day) - if len(gw.reports) != 1 { - t.Fatalf("gateway got %d reports", len(gw.reports)) - } - got := gw.reports[0] - if got.Date != "2026-10-23" || got.GridImport != 24 || got.VehicleCharge != 24 { - t.Fatalf("report %+v", got) - } - if kv[lastDayKey] != "2026-10-23" || gw.answered != http.StatusNoContent { - t.Fatalf("kv %v answered %d", kv, gw.answered) + got := gw.Energy() + if len(got) != 1 || got[0].Date != "2026-10-23" || got[0].GridImport != 24 || got[0].VehicleCharge != 24 { + t.Fatalf("gateway got %+v", got) } - body := relay.bodies[0] - if bytes.Contains(body, []byte("kwh")) || bytes.Contains(body, []byte("2026-10-23")) || len(body) != 7+32+energywire.RequestSize+16 { - t.Fatalf("the relay saw %d bytes: %q", len(body), body) + if kv[lastDayKey] != "2026-10-23" || gw.Answered() != http.StatusNoContent || len(relay.Bodies()) != 1 { + t.Fatalf("kv %v answered %d", kv, gw.Answered()) } - if relay.headers[0].Get("Content-Type") != ohttp.RequestType || relay.headers[0].Get("User-Agent") != "ftw-energy-report" { - t.Fatalf("relay headers %v", relay.headers[0]) - } - // A restart in the same window does not send the day again. s.send(context.Background(), day) - if len(gw.reports) != 1 { + if len(gw.Energy()) != 1 { t.Fatal("the same day went twice") } } func TestNothingIsSentWithoutTheOwnersAnswer(t *testing.T) { - g := testGateway(t) - gw := newFakeGateway(t, g) - relay := newFakeRelay(t, gw.URL) - cfg := g.Configs()[0] + client, _, relay := statsrelaytest.New(t, day.Add(30*time.Hour)) ledger := whole(t) var mu sync.Mutex reads := 0 @@ -146,9 +87,9 @@ func TestNothingIsSentWithoutTheOwnersAnswer(t *testing.T) { if o.Ledger == nil { o.Ledger = ledger } - o.Relay, o.Key, o.KV = relay.server.URL, &cfg, memKV{} + o.Relay, o.KV = client, memKV{} newSender(t, o).send(context.Background(), day) - if len(relay.bodies) != 0 { + if len(relay.Bodies()) != 0 { t.Fatalf("%s: sent", name) } } @@ -173,39 +114,6 @@ func TestWithoutARelayRunSendsNothing(t *testing.T) { } } -// The box believes only a sealed 204 from the gateway it encrypted to. -func TestTheBoxReadsOnlyTheGatewaysSealedAnswer(t *testing.T) { - g := testGateway(t) - cfg := g.Configs()[0] - ledger := whole(t) - r, _, _ := Day(context.Background(), ledger, day) - for name, h := range map[string]http.HandlerFunc{ - "relay error": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) }, - "plain 204": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }, - "forged body": func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", ohttp.ResponseType) - _, _ = w.Write(make([]byte, 64)) - }, - "sealed 400": sealedAnswer(t, g, http.StatusBadRequest), - "redirect": func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "https://elsewhere.example/", http.StatusTemporaryRedirect) - }, - } { - srv := httptest.NewServer(h) - s := newSender(t, Options{Ledger: ledger, Relay: srv.URL, Key: &cfg}) - if err := s.post(context.Background(), r); err == nil { - t.Fatalf("%s: accepted", name) - } - srv.Close() - } - srv := httptest.NewServer(sealedAnswer(t, g, http.StatusNoContent)) - defer srv.Close() - s := newSender(t, Options{Ledger: ledger, Relay: srv.URL, Key: &cfg}) - if err := s.post(context.Background(), r); err != nil { - t.Fatal(err) - } -} - func newSender(t *testing.T, o Options) *Sender { t.Helper() if o.Ledger == nil { @@ -218,93 +126,6 @@ func newSender(t *testing.T, o Options) *Sender { return s } -func testGateway(t *testing.T) *ohttp.Gateway { - t.Helper() - k, err := ecdh.X25519().GenerateKey(nil) - if err != nil { - t.Fatal(err) - } - g, err := ohttp.NewGateway(ohttp.Key{ID: 7, Private: k}) - if err != nil { - t.Fatal(err) - } - return g -} - -func sealedAnswer(t *testing.T, g *ohttp.Gateway, status int) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - _, ctx, err := g.Decapsulate(body) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - sealed, _ := ctx.Seal(ohttp.Response{Status: status}.Marshal(energywire.ResponseSize)) - w.Header().Set("Content-Type", ohttp.ResponseType) - _, _ = w.Write(sealed) - } -} - -// fakeGateway stands in for FTW Cloud's: it reads only the one inner request. -type fakeGateway struct { - *httptest.Server - g *ohttp.Gateway - reports []energywire.Report - answered int -} - -func newFakeGateway(t *testing.T, g *ohttp.Gateway) *fakeGateway { - f := &fakeGateway{g: g} - f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - msg, ctx, err := g.Decapsulate(body) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - f.answered = http.StatusBadRequest - req, err := ohttp.ParseRequest(msg) - if err == nil && req.Method == energywire.Method && req.Path == energywire.Path && len(msg) == energywire.RequestSize { - if rep, err := energywire.Decode(req.Content, day.Add(30*time.Hour)); err == nil { - f.reports = append(f.reports, rep) - f.answered = http.StatusNoContent - } - } - sealed, _ := ctx.Seal(ohttp.Response{Status: f.answered}.Marshal(energywire.ResponseSize)) - w.Header().Set("Content-Type", ohttp.ResponseType) - _, _ = w.Write(sealed) - })) - t.Cleanup(f.Close) - return f -} - -// fakeRelay forwards the body and content type, and nothing about the box. -type fakeRelay struct { - server *httptest.Server - bodies [][]byte - headers []http.Header -} - -func newFakeRelay(t *testing.T, gateway string) *fakeRelay { - f := &fakeRelay{} - f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - f.bodies = append(f.bodies, body) - f.headers = append(f.headers, r.Header.Clone()) - resp, err := http.Post(gateway, r.Header.Get("Content-Type"), bytes.NewReader(body)) - if err != nil { - w.WriteHeader(http.StatusBadGateway) - return - } - defer resp.Body.Close() - w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) - })) - t.Cleanup(f.server.Close) - return f -} - type memKV map[string]string func (m memKV) LoadConfig(k string) (string, bool) { v, ok := m[k]; return v, ok } diff --git a/go/internal/fleetping/fleetping.go b/go/internal/fleetping/fleetping.go index 812349f48..1a360ee5c 100644 --- a/go/internal/fleetping/fleetping.go +++ b/go/internal/fleetping/fleetping.go @@ -20,10 +20,13 @@ // - a send time that moves. A box that pings at 03:17 every night has told // the endpoint its name in a field nobody thought to look at. // -// What this cannot fix is the source IP, which the endpoint sees and which a -// household often keeps for months. That is the honest limit: this design -// makes the *payload* useless as an identifier, it does not make the -// connection anonymous. The drifting send time is what stops arrival time +// Sent straight to an endpoint, the source IP is what this cannot fix: the +// endpoint sees it, and a household often keeps one for months. A release +// that names a relay (package statsrelay) sends the report to FTW Cloud +// through it instead, sealed with Oblivious HTTP, so FTW Cloud reads the +// report without the address and the relay sees the address without the +// report. Without one, this design makes the *payload* useless as an +// identifier; it does not make the connection anonymous. The drifting send time is what stops arrival time // becoming a second selector alongside it — though the drift is a random walk // in six-hour steps, so it takes about four days to spread across the clock, // and until then one box's arrival times are still loosely related. @@ -53,6 +56,7 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/prices" + "github.com/srcfl/ftw/go/internal/statsrelay" "github.com/srcfl/ftw/go/protocol/fleetwire" ) @@ -347,6 +351,26 @@ func (h *HTTPProvider) Send(ctx context.Context, p Payload) error { return nil } +// RelayProvider sends the report through the relay (package statsrelay), as +// its own message at its own time: never beside the energy report, whose +// sums a driver set and a price zone could otherwise tie to one home. +type RelayProvider struct { + c *statsrelay.Client +} + +func NewRelayProvider(c *statsrelay.Client) *RelayProvider { return &RelayProvider{c: c} } + +func (p *RelayProvider) Send(ctx context.Context, payload Payload) error { + body, err := json.Marshal(payload) + if err != nil { + return err + } + return p.c.Send(ctx, body) +} + +// EndpointURL is the relay, which is where the box sends. +func (p *RelayProvider) EndpointURL() string { return p.c.Relay() } + // Options wires the pinger. Everything except the injectable seams is required. type Options struct { // Enabled is read at each send rather than captured, so turning the ping @@ -426,6 +450,16 @@ func (p *Pinger) Endpoint() string { return "" } +// Enabled is whether the next report would go, read the way the sender reads +// it, so Settings says what the box does. +func (p *Pinger) Enabled() bool { return p.enabled() } + +// ThroughRelay reports whether the running provider sends through the relay. +func (p *Pinger) ThroughRelay() bool { + _, ok := p.provider.(*RelayProvider) + return ok +} + // Run pings once a day until ctx ends. func (p *Pinger) Run(ctx context.Context) { for { diff --git a/go/internal/fleetping/fleetping_test.go b/go/internal/fleetping/fleetping_test.go index b0848ab9b..b5ecb5e2c 100644 --- a/go/internal/fleetping/fleetping_test.go +++ b/go/internal/fleetping/fleetping_test.go @@ -1,6 +1,7 @@ package fleetping import ( + "bytes" "context" "encoding/json" @@ -15,6 +16,7 @@ import ( "time" "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest" ) // The whole point of this package is what it does NOT send. These tests are @@ -542,3 +544,26 @@ func TestHTTPProviderDefaultClientHasATimeout(t *testing.T) { t.Fatal("the default client has no timeout") } } + +// Through the relay the gateway reads the same report the box would post +// directly, and the relay sees none of it. +func TestTheReportGoesThroughTheRelayUnchanged(t *testing.T) { + client, gw, relay := statsrelaytest.New(t, time.Now()) + p := NewRelayProvider(client) + facts := Facts{Version: "v1.4.0", Drivers: []string{"sungrow"}, Catalogue: []string{"sungrow"}, PriceZone: "SE3"} + pinger, err := New(Options{Facts: func() Facts { return facts }, Provider: p}) + if err != nil { + t.Fatal(err) + } + want := pinger.Payload() + if err := p.Send(context.Background(), want); err != nil { + t.Fatal(err) + } + got := gw.Fleet() + if len(got) != 1 || got[0].Version != want.Version || got[0].Zone != "SE3" || len(got[0].Drivers) != 1 { + t.Fatalf("gateway got %+v", got) + } + if bytes.Contains(relay.Bodies()[0], []byte("sungrow")) || !pinger.ThroughRelay() || pinger.Endpoint() != relay.URL { + t.Fatal("the relay saw the report, or the pinger does not say where it sends") + } +} diff --git a/go/internal/statsrelay/statsrelay.go b/go/internal/statsrelay/statsrelay.go new file mode 100644 index 000000000..5112224a4 --- /dev/null +++ b/go/internal/statsrelay/statsrelay.go @@ -0,0 +1,145 @@ +// Package statsrelay is how the box's anonymous daily reports leave it once a +// release names a relay: each report alone, sealed with Oblivious HTTP +// (RFC 9458) to FTW Cloud's gateway key and posted to a relay that someone +// other than Sourceful runs. The relay sees the box's address but not the +// report; FTW Cloud reads the report but not the address. +// +// Two reports use it, the fleet report (package fleetping) and the energy +// report (package energyreport), each at its own random time and never in +// one message. +package statsrelay + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "time" + + "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" +) + +// Compile-time settings, as in cloudlink: a relay a household could point +// elsewhere is a relay a support call could point at an attacker's. A local +// test build may set them with -ldflags -X. +var ( + // RelayURL is the OHTTP relay, run by someone other than Sourceful. + // Empty until that relay exists: the energy report is then not sent, + // and the fleet report goes straight to FTW Cloud as before. + RelayURL = "" + // GatewayKey is FTW Cloud's gateway key configuration (RFC 9458, + // section 3), base64url, as + // https://cloud.ftw.energy/.well-known/ohttp-gateway serves it. The box + // encrypts only to this key and never asks the service for one, so every + // box on a release uses the same key and a key swapped in at the service + // reads nothing. Empty means no gateway key exists for this build. + GatewayKey = "" +) + +const ( + sendTimeout = 30 * time.Second + maxResponseBytes = 1 << 10 +) + +// Client posts sealed reports to one relay under one gateway key. +type Client struct { + relay string + key ohttp.KeyConfig + http *http.Client +} + +// Pinned is this build's client, or nil when the build names no usable relay +// and gateway key. +func Pinned() *Client { + relay, key, ok := pinned(RelayURL, GatewayKey) + if !ok { + return nil + } + return New(relay, key, nil) +} + +func pinned(relay, key string) (string, ohttp.KeyConfig, bool) { + raw, err := base64.RawURLEncoding.DecodeString(key) + if err != nil || relay == "" || key == "" { + return "", ohttp.KeyConfig{}, false + } + cfg, err := ohttp.ParseKeyConfig(raw) + if err != nil { + return "", ohttp.KeyConfig{}, false + } + u, err := url.Parse(relay) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == reportwire.Authority { + return "", ohttp.KeyConfig{}, false + } + return relay, cfg, true +} + +// New is a client for one relay and key. Tests point it at a fake relay. +func New(relay string, key ohttp.KeyConfig, client *http.Client) *Client { + if client == nil { + client = &http.Client{Timeout: sendTimeout} + } else { + c := *client + client = &c + } + // A redirect would resend the sealed report to an address nobody chose. + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + return &Client{relay: relay, key: key, http: client} +} + +// Relay is the relay's URL, for Settings. +func (c *Client) Relay() string { return c.relay } + +// Send seals one report body and reads the gateway's answer. Only a sealed +// 204 from the gateway the box encrypted to counts as delivered. +func (c *Client) Send(ctx context.Context, body []byte) error { + inner, ok := reportwire.Request(body) + if !ok { + return errors.New("statsrelay: the report does not fit the padded request") + } + sealed, reader, err := ohttp.Encapsulate(c.key, inner) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, sendTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.relay, bytes.NewReader(sealed)) + if err != nil { + return err + } + req.Header.Set("Content-Type", ohttp.RequestType) + // Pinned, as for the fleet report's direct path: the relay sees it, and + // Go's default names the runtime's version. + req.Header.Set("User-Agent", "ftw-report") + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil { + return err + } + media, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if resp.StatusCode != http.StatusOK || media != ohttp.ResponseType || len(data) > maxResponseBytes { + return fmt.Errorf("statsrelay: relay answered %d %q", resp.StatusCode, media) + } + msg, err := reader.Open(data) + if err != nil { + return err + } + answer, err := ohttp.ParseResponse(msg) + if err != nil { + return err + } + if answer.Status != http.StatusNoContent { + return fmt.Errorf("statsrelay: gateway answered %d", answer.Status) + } + return nil +} diff --git a/go/internal/statsrelay/statsrelay_test.go b/go/internal/statsrelay/statsrelay_test.go new file mode 100644 index 000000000..8df367a59 --- /dev/null +++ b/go/internal/statsrelay/statsrelay_test.go @@ -0,0 +1,131 @@ +package statsrelay_test + +import ( + "bytes" + "context" + "crypto/ecdh" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/statsrelay" + "github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" +) + +var now = time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC) + +// The release names no relay and no gateway key yet, so no report goes +// through one. Once both are pinned they must be a usable pair. +func TestTheReleasePinsBothOrNeither(t *testing.T) { + c := statsrelay.Pinned() + switch { + case statsrelay.RelayURL == "" && statsrelay.GatewayKey == "": + if c != nil { + t.Fatal("nothing pinned, yet a client") + } + case c == nil: + t.Fatalf("the pinned relay %q and gateway key do not make a usable pair", statsrelay.RelayURL) + } +} + +func TestAPinIsRefusedUnlessItIsARelayAndAKey(t *testing.T) { + k, _ := ecdh.X25519().GenerateKey(nil) + gw, _ := ohttp.NewGateway(ohttp.Key{ID: 1, Private: k}) + key := base64.RawURLEncoding.EncodeToString(gw.Configs()[0].Marshal()) + oldRelay, oldKey := statsrelay.RelayURL, statsrelay.GatewayKey + t.Cleanup(func() { statsrelay.RelayURL, statsrelay.GatewayKey = oldRelay, oldKey }) + set := func(relay, key string) *statsrelay.Client { + statsrelay.RelayURL, statsrelay.GatewayKey = relay, key + return statsrelay.Pinned() + } + if c := set("https://relay.example.net/ftw", key); c == nil || c.Relay() != "https://relay.example.net/ftw" { + t.Fatal("a good pair was refused") + } + for name, p := range map[string][2]string{ + "no relay": {"", key}, + "no key": {"https://relay.example.net/ftw", ""}, + "plain http": {"http://relay.example.net/ftw", key}, + "FTW Cloud itself": {"https://cloud.ftw.energy/v1/ohttp", key}, + "query": {"https://relay.example.net/ftw?box=1", key}, + "credentials": {"https://a:b@relay.example.net/ftw", key}, + "bad key": {"https://relay.example.net/ftw", "AAAA"}, + "padded base64": {"https://relay.example.net/ftw", key + "="}, + } { + if set(p[0], p[1]) != nil { + t.Fatalf("%s was accepted", name) + } + } +} + +// The relay sees one length and no report; the gateway reads it. +func TestAReportReachesTheGatewayThroughTheRelay(t *testing.T) { + client, gw, relay := statsrelaytest.New(t, now) + body, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", GridImport: 24}) + if err := client.Send(context.Background(), body); err != nil { + t.Fatal(err) + } + if got := gw.Energy(); len(got) != 1 || got[0].GridImport != 24 { + t.Fatalf("gateway got %+v", got) + } + seen := relay.Bodies()[0] + if bytes.Contains(seen, []byte("kwh")) || len(seen) != 7+32+reportwire.RequestSize+16 { + t.Fatalf("the relay saw %d bytes", len(seen)) + } + h := relay.Headers()[0] + if h.Get("Content-Type") != ohttp.RequestType || h.Get("User-Agent") != "ftw-report" { + t.Fatalf("relay headers %v", h) + } + if err := client.Send(context.Background(), make([]byte, reportwire.RequestSize)); err == nil { + t.Fatal("a report too large for the padding went out") + } +} + +// The box believes only a sealed 204 from the gateway it encrypted to. +func TestOnlyTheGatewaysSealed204Counts(t *testing.T) { + k, _ := ecdh.X25519().GenerateKey(nil) + gw, _ := ohttp.NewGateway(ohttp.Key{ID: 1, Private: k}) + sealed := func(status int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _, ctx, err := gw.Decapsulate(body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + answer, _ := ctx.Seal(reportwire.Response(status)) + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(answer) + } + } + body := []byte(`{"schema":"x"}`) + for name, h := range map[string]http.HandlerFunc{ + "relay error": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) }, + "plain 204": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }, + "forged body": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(make([]byte, 64)) + }, + "sealed 400": sealed(http.StatusBadRequest), + "redirect": func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://elsewhere.example/", http.StatusTemporaryRedirect) + }, + } { + srv := httptest.NewServer(h) + if err := statsrelay.New(srv.URL, gw.Configs()[0], nil).Send(context.Background(), body); err == nil { + t.Fatalf("%s: accepted", name) + } + srv.Close() + } + srv := httptest.NewServer(sealed(http.StatusNoContent)) + defer srv.Close() + if err := statsrelay.New(srv.URL, gw.Configs()[0], nil).Send(context.Background(), body); err != nil { + t.Fatal(err) + } +} diff --git a/go/internal/statsrelay/statsrelaytest/statsrelaytest.go b/go/internal/statsrelay/statsrelaytest/statsrelaytest.go new file mode 100644 index 000000000..1d6a66b1b --- /dev/null +++ b/go/internal/statsrelay/statsrelaytest/statsrelaytest.go @@ -0,0 +1,145 @@ +// Package statsrelaytest is a fake relay and gateway for tests. The gateway +// reads a report as FTW Cloud's does: one padded inner request, its decoder +// chosen by schema. It keeps the reports for the test to read. +package statsrelaytest + +import ( + "bytes" + "crypto/ecdh" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/statsrelay" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" + "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" +) + +// Gateway is a fake FTW Cloud gateway. +type Gateway struct { + *httptest.Server + Key ohttp.KeyConfig + + mu sync.Mutex + now time.Time + fleet []fleetwire.Report + energy []energywire.Report + last int +} + +// Relay forwards only the body and its content type, and records what it saw. +type Relay struct { + *httptest.Server + + mu sync.Mutex + bodies [][]byte + headers []http.Header +} + +// New starts a gateway whose clock reads now, a relay in front of it, and a +// client posting to the relay. +func New(t testing.TB, now time.Time) (*statsrelay.Client, *Gateway, *Relay) { + t.Helper() + k, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + gw, err := ohttp.NewGateway(ohttp.Key{ID: 7, Private: k}) + if err != nil { + t.Fatal(err) + } + g := &Gateway{Key: gw.Configs()[0], now: now} + g.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + msg, sealer, err := gw.Decapsulate(body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + status := g.add(msg) + sealed, _ := sealer.Seal(reportwire.Response(status)) + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(sealed) + })) + t.Cleanup(g.Close) + + rl := &Relay{} + rl.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + rl.mu.Lock() + rl.bodies = append(rl.bodies, body) + rl.headers = append(rl.headers, r.Header.Clone()) + rl.mu.Unlock() + resp, err := http.Post(g.URL, r.Header.Get("Content-Type"), bytes.NewReader(body)) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + defer resp.Body.Close() + w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + t.Cleanup(rl.Close) + return statsrelay.New(rl.URL, g.Key, nil), g, rl +} + +func (g *Gateway) add(msg []byte) int { + g.mu.Lock() + defer g.mu.Unlock() + g.last = http.StatusBadRequest + body, ok := reportwire.Body(msg) + if !ok { + return g.last + } + switch reportwire.Schema(body) { + case fleetwire.Schema: + if r, err := fleetwire.Decode(body); err == nil { + g.fleet, g.last = append(g.fleet, r), http.StatusNoContent + } + case energywire.Schema: + if r, err := energywire.Decode(body, g.now); err == nil { + g.energy, g.last = append(g.energy, r), http.StatusNoContent + } + } + return g.last +} + +// Fleet is every fleet report the gateway took. +func (g *Gateway) Fleet() []fleetwire.Report { + g.mu.Lock() + defer g.mu.Unlock() + return append([]fleetwire.Report(nil), g.fleet...) +} + +// Energy is every energy report the gateway took. +func (g *Gateway) Energy() []energywire.Report { + g.mu.Lock() + defer g.mu.Unlock() + return append([]energywire.Report(nil), g.energy...) +} + +// Answered is the gateway's last inner status. +func (g *Gateway) Answered() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.last +} + +// Bodies is every body the relay forwarded, and Headers what came with each. +func (r *Relay) Bodies() [][]byte { + r.mu.Lock() + defer r.mu.Unlock() + return append([][]byte(nil), r.bodies...) +} + +func (r *Relay) Headers() []http.Header { + r.mu.Lock() + defer r.mu.Unlock() + return append([]http.Header(nil), r.headers...) +} diff --git a/go/protocol/energywire/energy.go b/go/protocol/energywire/energy.go index 07af21371..08c95fd89 100644 --- a/go/protocol/energywire/energy.go +++ b/go/protocol/energywire/energy.go @@ -1,10 +1,10 @@ // Package energywire names the anonymous daily energy report: one closed UTC // day of one home as six whole-kWh sums, and nothing that says whose. // -// It travels inside Oblivious HTTP (package ohttp) through a relay, so the -// gateway never sees the box's address. It carries no id, version, driver, -// zone or size, because the daily fleet report (package fleetwire) carries -// those, and the two must never be joined into one home's record. +// It travels alone through a relay (package reportwire), so the gateway never +// sees the box's address. It carries no id, version, driver, zone or size, +// because the daily fleet report (package fleetwire) carries those, and the +// two must never be joined into one home's record. package energywire import ( @@ -16,21 +16,7 @@ import ( const Schema = "ftw.energy/1" -// The one inner request a box sends. The gateway answers no other. const ( - Method = "POST" - Scheme = "https" - Authority = "cloud.ftw.energy" - Path = "/v1/energy/report" -) - -const ( - // RequestSize is every padded inner request, and ResponseSize every - // padded inner response, so the relay sees one length whatever the - // numbers or the answer. - RequestSize = 320 - ResponseSize = 16 - // MaxBodyBytes bounds the report itself, before any parsing. MaxBodyBytes = 256 diff --git a/go/protocol/energywire/energy_test.go b/go/protocol/energywire/energy_test.go index 08b60911f..0b6222ab4 100644 --- a/go/protocol/energywire/energy_test.go +++ b/go/protocol/energywire/energy_test.go @@ -5,8 +5,6 @@ import ( "strings" "testing" "time" - - "github.com/srcfl/ftw/go/protocol/ohttp" ) var now = time.Date(2026, 10, 24, 9, 30, 0, 0, time.UTC) @@ -71,22 +69,3 @@ func TestOpenAndClosedDays(t *testing.T) { } } } - -// The largest report still fits the padded request, so every request has the -// same length on the wire. -func TestTheLargestReportFitsThePaddedRequest(t *testing.T) { - r := Report{Schema: Schema, Date: "2026-10-23", PVGeneration: MaxKWh, GridImport: MaxKWh, GridExport: MaxKWh, - BatteryCharge: MaxKWh, BatteryDischarge: MaxKWh, VehicleCharge: MaxKWh} - body, _ := json.Marshal(r) - if len(body) > MaxBodyBytes { - t.Fatalf("body %d", len(body)) - } - req := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: body}.Marshal(RequestSize) - small := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: []byte(valid)}.Marshal(RequestSize) - if len(req) != RequestSize || len(small) != RequestSize { - t.Fatalf("padded %d and %d", len(req), len(small)) - } - if len(ohttp.Response{Status: 400}.Marshal(ResponseSize)) != ResponseSize { - t.Fatal("response padding") - } -} diff --git a/go/protocol/reportwire/report.go b/go/protocol/reportwire/report.go new file mode 100644 index 000000000..efafc0cd4 --- /dev/null +++ b/go/protocol/reportwire/report.go @@ -0,0 +1,68 @@ +// Package reportwire is how the box's anonymous daily reports reach FTW +// Cloud: each report alone, as one padded inner request sealed with Oblivious +// HTTP (package ohttp) and posted to an independent relay. The gateway reads +// the report's schema and hands it to that report's decoder: the fleet report +// (package fleetwire) or the energy report (package energywire). +// +// Two reports, never one message. With a fleet of a few boxes a driver set +// and a price zone can point at one home, so energy sent beside them would +// belong to that home again. Each report leaves at its own random time. +package reportwire + +import ( + "encoding/json" + + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// The one inner request. The gateway answers no other. +const ( + Method = "POST" + Scheme = "https" + Authority = "cloud.ftw.energy" + Path = "/v1/report" +) + +const ( + // RequestSize is every padded inner request and ResponseSize every padded + // inner response, so the relay sees one length whatever the report, its + // kind or the answer. The largest fleet report fits. + RequestSize = 1536 + ResponseSize = 16 +) + +// Request is the padded inner request carrying one report body. ok is false +// when the body does not fit, so a report never goes out at a length of its +// own. +func Request(body []byte) (msg []byte, ok bool) { + msg = ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: body}.Marshal(RequestSize) + return msg, len(msg) == RequestSize +} + +// Body opens an inner request and returns the report it carries, or false +// for anything but the one padded request. +func Body(msg []byte) ([]byte, bool) { + req, err := ohttp.ParseRequest(msg) + if err != nil || len(msg) != RequestSize || req.Method != Method || req.Scheme != Scheme || + req.Authority != Authority || req.Path != Path || len(req.Fields) != 0 { + return nil, false + } + return req.Content, true +} + +// Schema reads only a report's schema, to choose its decoder. The decoder +// still checks every byte. +func Schema(body []byte) string { + var head struct { + Schema string `json:"schema"` + } + if json.Unmarshal(body, &head) != nil { + return "" + } + return head.Schema +} + +// Response is the padded inner answer. +func Response(status int) []byte { + return ohttp.Response{Status: status}.Marshal(ResponseSize) +} diff --git a/go/protocol/reportwire/report_test.go b/go/protocol/reportwire/report_test.go new file mode 100644 index 000000000..a4c63f8bd --- /dev/null +++ b/go/protocol/reportwire/report_test.go @@ -0,0 +1,68 @@ +package reportwire + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// The largest report of either kind fills the same padded request, so the +// relay cannot tell a fleet report from an energy report, or one box's from +// another's, by its length. +func TestEveryReportHasOneLength(t *testing.T) { + drivers := make([]string, 16) + for i := range drivers { + drivers[i] = strings.Repeat(string(rune('a'+i)), 64) + } + fleet, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v1234.1234.1234-beta.1234", Channel: "unknown", + Drivers: drivers, Battery: "30+", Zone: strings.Repeat("Z", 16), Age: "unknown"}) + energy, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", PVGeneration: 1000, GridImport: 1000, + GridExport: 1000, BatteryCharge: 1000, BatteryDischarge: 1000, VehicleCharge: 1000}) + small, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23"}) + for _, body := range [][]byte{fleet, energy, small} { + msg, ok := Request(body) + if !ok || len(msg) != RequestSize { + t.Fatalf("%d-byte report padded to %d", len(body), len(msg)) + } + got, ok := Body(msg) + if !ok || string(got) != string(body) || Schema(got) == "" { + t.Fatalf("read back %q", got) + } + } + if Schema(fleet) != fleetwire.Schema || Schema(energy) != energywire.Schema || Schema([]byte("x")) != "" { + t.Fatal("schema") + } + if _, ok := Request(make([]byte, RequestSize)); ok { + t.Fatal("an oversized report was padded") + } + if len(Response(204)) != ResponseSize { + t.Fatal("response padding") + } +} + +func TestOnlyTheOneInnerRequestIsRead(t *testing.T) { + good := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: []byte("{}")} + other := func(f func(*ohttp.Request)) []byte { + r := good + f(&r) + return r.Marshal(RequestSize) + } + for name, msg := range map[string][]byte{ + "another path": other(func(r *ohttp.Request) { r.Path = "/v1/energy/report" }), + "another method": other(func(r *ohttp.Request) { r.Method = "PUT" }), + "another host": other(func(r *ohttp.Request) { r.Authority = "example.com" }), + "plain http": other(func(r *ohttp.Request) { r.Scheme = "http" }), + "a header": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "x-home", Value: "1"}} }), + "not padded": good.Marshal(0), + "padded more": good.Marshal(RequestSize + 1), + "a response": Response(200), + } { + if _, ok := Body(msg); ok { + t.Fatalf("%s was read", name) + } + } +} diff --git a/web/app-link-tab.test.mjs b/web/app-link-tab.test.mjs index 5dca9e744..4b798b00b 100644 --- a/web/app-link-tab.test.mjs +++ b/web/app-link-tab.test.mjs @@ -91,16 +91,16 @@ describe("the app tab", () => { assert.match(render({}), /id="app-link-pair"[^>]*disabled/); }); - it("keeps fleet statistics with the app, but independent of its switch", () => { + it("keeps statistics with the app, but independent of its switch", () => { const appWithFleet = loadTab(true); const config = { app_link: { enabled: false } }; const html = appWithFleet.render({ config }); - assert.match(html, /Anonymous daily report<\/legend>/); - assert.match(html, /data-checkbox-path="fleet_ping\.enabled"/); - assert.equal(config.fleet_ping.enabled, true, "fleet sharing should still default on"); + assert.match(html, /Anonymous statistics<\/legend>/); + assert.match(html, /id="statistics-share"/); + assert.doesNotMatch(html, /data-checkbox-path="fleet_ping\.enabled"/); + assert.equal(config.fleet_ping, undefined, "statistics are not a config field"); assert.doesNotMatch(html, /id="app-link-enabled"[^>]*checked/); - assert.match(html, /id="fleet-ping-enabled"[^>]*checked/); }); it("refreshes the fleet payload after saving from the app page", () => { @@ -119,7 +119,7 @@ describe("the app tab", () => { vm.runInContext(fleetSource, sandbox); win.FTWSettings.tabs.app.afterSave(); - assert.deepEqual(asked, ["/api/fleet-ping", "/api/app-link/status"]); + assert.deepEqual([...asked].sort(), ["/api/app-link/status", "/api/cloud", "/api/cloud/stats-policy", "/api/fleet-ping"]); }); it("does not leave a standalone Fleet ping destination behind", () => { diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs index 6a5824f1e..10db8f931 100644 --- a/web/fleet-ping-tab.test.mjs +++ b/web/fleet-ping-tab.test.mjs @@ -55,34 +55,21 @@ const payload = { }; describe("the fleet statistics panel", () => { - it("offers a checkbox bound to fleet_ping.enabled", () => { - const html = render({}); - assert.match(html, /data-checkbox-path="fleet_ping\.enabled"/); - assert.match(html, /type="checkbox"/); - }); - - it("reflects the saved setting", () => { - assert.doesNotMatch(render({ fleet_ping: { enabled: false } }), /checkbox"[^>]*checked/); - assert.match(render({ fleet_ping: { enabled: true } }), /checked/); - }); - - it("mirrors the box's default when the section is missing", () => { - // A box that has never had fleet_ping in its YAML is sending, because - // applyDefaults turns it on. An unchecked box here would say the opposite - // of what is true, which is the one thing this screen must never do. + it("offers one statistics switch, not a config setting", () => { + // Statistics are one answer for both reports, kept by the box, not a + // field in the config document that Save writes. const config = {}; const html = render(config); - // Compared field by field: the object is built inside the vm sandbox, so - // its prototype comes from another realm and deepStrictEqual refuses it. - assert.ok(config.fleet_ping, "fleet_ping was not created"); - assert.equal(config.fleet_ping.enabled, true); - assert.match(html, /checked/); + assert.match(html, /id="statistics-share"/); + assert.doesNotMatch(html, /data-checkbox-path/); + assert.equal(config.fleet_ping, undefined, "render() wrote a config section"); }); - it("says the switch needs no restart", () => { - // The one thing about this setting that differs from the FTW app tab - // next to it, where saving is not enough. - assert.match(render({}), /[Nn]o restart/); + it("names both reports and says they never travel together", () => { + const html = render({}); + assert.match(html, /two anonymous reports a day/); + assert.match(html, /never travel together/); + assert.match(html, /part of Cloud/); }); it("names every field the message carries", () => { @@ -200,6 +187,36 @@ describe("the line under the payload", () => { }); }); +describe("the statistics switch", () => { + const { shareLine } = tab._pure; + + it("is part of FTW Cloud while Cloud is on", () => { + const line = shareLine({ statistics: true, statisticsRequired: true }); + assert.match(line, /Part of FTW Cloud/); + assert.match(line, /no separate off switch/); + }); + + it("says what on and off mean without Cloud", () => { + assert.match(shareLine({ statistics: true, statisticsRequired: false }), /takes effect at once, with no restart/); + assert.match(shareLine({ statistics: false, statisticsRequired: false }), /^Off\. Nothing is sent/); + assert.match(shareLine(null), /Could not read/); + }); + + it("does not claim a report goes while it waits for the relay", () => { + const line = whereLine({ enabled: false, waits_for_relay: true, endpoint: "https://cloud.ftw.energy/v1/fleet", ftw_relay: true, payload }); + assert.match(line, /^Not sent yet/); + assert.match(line, /only through an independent relay/); + assert.doesNotMatch(line, /^Sent/); + }); + + it("says who sees what when the report goes through the relay", () => { + const line = whereLine({ enabled: true, endpoint: "https://relay.example.net/ftw", ftw_relay: true, through_relay: true, payload }); + assert.match(line, /^Sent through https:\/\/relay\.example\.net\/ftw, an independent relay, to FTW Cloud/); + assert.match(line, /sees this box's address but not the report/); + assert.match(line, /reads the report but not the address/); + }); +}); + describe("keeping the line true after a save", () => { it("registers a post-save hook, because render() runs only on a tab switch", () => { // Without this the line keeps whatever the box said when the tab opened: @@ -222,7 +239,8 @@ describe("keeping the line true after a save", () => { vm.createContext(sandbox); vm.runInContext(source, sandbox); sandbox.window.FTWSettings.fleetPing.afterSave(); - assert.deepEqual(asked, ["/api/fleet-ping"]); + // And the statistics answer, which the switch shows. + assert.deepEqual([...asked].sort(), ["/api/cloud", "/api/cloud/stats-policy", "/api/fleet-ping"]); }); }); diff --git a/web/project-statistics.js b/web/project-statistics.js index 6948d51e7..f7abbbf9f 100644 --- a/web/project-statistics.js +++ b/web/project-statistics.js @@ -1,4 +1,6 @@ -// One owner choice: statistics come with Cloud; local use asks first. +// FTW Cloud's terms. Statistics are part of Cloud; a box without Cloud shares +// anonymous statistics until its owner turns them off under Anonymous +// statistics (settings/tabs/fleet.js), so nobody is asked here any more. (function (window, document) { "use strict"; var read = window.fetch.bind(window); @@ -35,11 +37,23 @@ function write(method,path,body) { return window.fetch(path,{method:method,headers:{"Content-Type":"application/json"},body:body?JSON.stringify(body):undefined}).then(function(r){return r.json().then(function(v){if(!r.ok)throw Error(v.error||"The box refused the choice.");justAnswered=true;return v;});}); } + // What FTW Cloud's statistics add, for a panel that offers or holds Cloud. + function cloudScope(parent) { + say(parent,"Shares recorded hourly energy and configured system capacities. Reports stay for 48 hours; public totals require at least 20 homes."); + var disclosure=node("details",null,parent); + node("summary","What is shared and how it is used",disclosure); + say(disclosure,details); + say(disclosure,privacy); + var link=node("a","Privacy and Cloud terms",parent);link.href="https://ftw.sourceful.energy/privacy#statistics";link.target="_blank";link.rel="noopener"; + if(!policy.available) say(parent,"You can save your choice now. Hourly reports wait for privacy review and activation."); + } function paint(host,settings) { host.textContent=""; - if (!status || !policy || (!settings && !status.statisticsDecisionRequired && !justAnswered)) {host.hidden=true;return;} + // Only a Cloud box under older terms has something to answer. + var needsTerms=!!status && status.link==="on" && !status.statisticsOnly && !status.cloudTermsAccepted; + if (!status || !policy || (!settings && !needsTerms && !justAnswered)) {host.hidden=true;return;} host.hidden=false; - if(!settings && !status.statisticsDecisionRequired && justAnswered) { + if(!settings && !needsTerms && justAnswered) { var saved=status.link==="ending"?"Stopping Cloud and deleting its stored copy. The box continues locally.":status.cloudTermsAccepted?"Cloud terms accepted and saved on this box.":status.statisticsActive?"Statistics choice saved. Change it in Settings → FTW app.":"Statistics sharing is off. The box continues locally."; var receipt=say(host,saved);receipt.className="project-statistics-saved";receipt.setAttribute("role","status"); return; @@ -48,26 +62,21 @@ panel.className="project-statistics-panel"; var assistant=host.id === "access-cloud-settings"; var cloud=status.link==="on" && !status.statisticsOnly; - node("legend",assistant?"FTW Cloud for assistants":cloud?(status.cloudTermsAccepted?"FTW Cloud":"Review FTW Cloud terms"):"Contribute to FTW statistics?",panel); + node("legend",assistant?"FTW Cloud for assistants":cloud?(status.cloudTermsAccepted?"FTW Cloud":"Review FTW Cloud terms"):"FTW Cloud",panel); if(assistant && !cloud) say(panel,"FTW Cloud lets your assistant reach this box. Sourceful can read its Cloud copy of your energy data. Cloud includes project statistics. Review the terms below before accepting."); else if(cloud) say(panel,status.cloudTermsAccepted?"The current Cloud terms are accepted. FTW Cloud includes project statistics.":"FTW Cloud includes project statistics. Sourceful can read its copy of your energy data. Accept the updated terms for the app and remote access, or turn Cloud off and keep using the box locally."); - else say(panel,"For local FTW use, sharing is your choice. Yes is the suggested choice. Nothing starts until you answer, and saying no leaves local operation working. Statistics-only sharing gives no app, phone or assistant remote access."); - say(panel,"Shares recorded hourly energy and configured system capacities. Reports stay for 48 hours; public totals require at least 20 homes."); - var disclosure=node("details",null,panel); - node("summary","What is shared and how it is used",disclosure); - say(disclosure,details); - say(disclosure,privacy); - var link=node("a","Privacy and Cloud terms",panel);link.href="https://ftw.sourceful.energy/privacy#statistics";link.target="_blank";link.rel="noopener"; - if(!policy.available) say(panel,"You can save your choice now. Hourly reports wait for privacy review and activation."); - if(!status.cloudTermsAccepted) { - say(panel,authStatus && authStatus.home_assistant + else say(panel,"FTW Cloud is off, and the box works locally. The anonymous statistics it shares, and the switch for them, are under Anonymous statistics below."); + if(cloud || assistant) cloudScope(panel); + var signIn=function(parent){ + say(parent,authStatus && authStatus.home_assistant ?"Home Assistant manages sign-in for this choice." :authStatus && authStatus.lan_auth ?"Your house password protects this choice. The box will ask you to log in if needed." :authStatus && authStatus.lan_auth===false ?"No house password is required. With password protection off, anyone who can reach this box on your private network can save this choice." :"This choice uses the same sign-in rules as Settings. A house password is optional."); - } + }; + if(!status.cloudTermsAccepted && (cloud || assistant)) signIn(panel); var actions=node("div",null,panel);actions.className="app-link-actions"; var feedback=node("p",null,panel);feedback.className="project-statistics-error";feedback.setAttribute("role","alert");feedback.hidden=true; if(status.link==="ending") {say(panel,"Cloud is stopping and deleting its stored copy. Refresh after the box receives the deletion receipt.");return;} @@ -80,22 +89,12 @@ },false,feedback); } else if(assistant) { button(actions,"Accept Cloud terms and turn it on",function(){return write("POST","/api/cloud/grant",{scopes:policy.cloud_scopes,policy:policy.cloud_policy});},true,feedback); - var local=node("details",null,panel);node("summary","Statistics without assistant remote access",local); - say(local,"Statistics-only sharing gives no app, phone or assistant remote access."); - if(status.statisticsOnly) button(local,"Stop sharing statistics",function(){return write("DELETE","/api/cloud/stats");},false,feedback); - else {button(local,"Yes, contribute",function(){return write("POST","/api/cloud/stats",{policy:policy.policy});},false,feedback);} - } else if(status.statisticsOnly) { - say(panel,"Local statistics sharing is on. Remote Cloud functions are off."); - button(actions,"Stop sharing statistics",function(){return write("DELETE","/api/cloud/stats");},false,feedback); } else { - button(actions,"Yes, contribute",function(){return write("POST","/api/cloud/stats",{policy:policy.policy});},true,feedback); - if(status.statisticsDecisionRequired) button(actions,"No, keep it local",function(){return write("DELETE","/api/cloud/stats");},false,feedback); - else say(panel,"Local statistics sharing is off."); - } - if(settings && !cloud && !assistant) { var more=node("details",null,panel); node("summary","Use FTW Cloud and the app",more); - say(more,"FTW Cloud adds app and assistant remote access and a readable state copy. Statistics are included for every Cloud user. Read the information above before accepting."); + say(more,"FTW Cloud adds app and assistant remote access and a readable state copy. Statistics are part of Cloud, with no separate off switch."); + cloudScope(more); + signIn(more); button(more,"Accept Cloud terms and turn it on",function(){return write("POST","/api/cloud/grant",{scopes:policy.cloud_scopes,policy:policy.cloud_policy});},true,feedback); } } diff --git a/web/project-statistics.test.mjs b/web/project-statistics.test.mjs index 53ebff7bc..15ba6223d 100644 --- a/web/project-statistics.test.mjs +++ b/web/project-statistics.test.mjs @@ -39,21 +39,20 @@ function page(cloud=false,authorised=true,options={}) { return {window,writes,main,all,buttons:()=>all().filter(e=>e.tag==='button')}; } -test('local question sends nothing before a real Yes or No',async()=>{ +test('a box without Cloud is not asked, and Settings offers Cloud only',async()=>{ const p=page();await p.window.FTWStatistics.refresh(); assert.equal(p.writes.length,0); - assert.deepEqual(p.buttons().map(e=>e.textContent),['Yes, contribute','No, keep it local']); - assert.equal(p.buttons()[0].className,'btn-send'); - p.buttons()[1].events.click();await setImmediate(); - assert.equal(p.writes.length,1);assert.equal(p.writes[0].method,'DELETE'); - assert.equal(p.writes[0].path,'/api/cloud/stats'); - assert.match(p.main.textContent,/sharing is off/); -}); - -test('local Yes grants only statistics, not Cloud access',async()=>{ - const p=page();await p.window.FTWStatistics.refresh();p.buttons()[0].events.click();await setImmediate(); - assert.equal(p.writes.length,1);assert.equal(p.writes[0].path,'/api/cloud/stats'); - assert.deepEqual(JSON.parse(p.writes[0].body),{policy:'stats-current'}); + assert.equal(p.buttons().length,0); + const prompt=p.all().find(e=>e.id==='project-statistics-prompt'); + assert.equal(prompt.hidden,true); + const settings={tag:'div',children:[],events:{},style:{},attributes:{},ownText:'',appendChild(e){this.children.push(e);return e;},set textContent(v){this.children=[];},get textContent(){return this.children.map(e=>e.textContent).join(' ');}}; + p.window.FTWStatistics.mount(settings); + const labels=[];const visit=e=>{if(e.tag==='button')labels.push(e.textContent);(e.children||[]).forEach(visit);};visit(settings); + assert.deepEqual(labels,['Accept Cloud terms and turn it on']); + assert.match(settings.textContent,/FTW Cloud is off/); + assert.match(settings.textContent,/Anonymous statistics below/); + assert.doesNotMatch(settings.textContent,/Yes, contribute|keep it local/); + assert.equal(p.writes.length,0); }); test('Cloud has bundled terms and a whole-service off action',async()=>{ diff --git a/web/settings/tabs/app.js b/web/settings/tabs/app.js index d077f03ea..805f72b2f 100644 --- a/web/settings/tabs/app.js +++ b/web/settings/tabs/app.js @@ -506,8 +506,8 @@ S.tabs.app = { afterSave: function () { - // These two switches share a page, not a state. The app link has its - // own restart boundary; fleet sharing takes effect as soon as it saves. + // These share a page, not a state. The app link has its own restart + // boundary; the statistics switch writes at once. var panel = S.fleetPing; if (panel && panel.afterSave) panel.afterSave(); refreshStatus(pairingCtx); diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 080cc2671..1aeade10c 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -1,4 +1,5 @@ -// Settings → Fleet ping: the message this box adds to FTW's fleet totals. +// Settings → FTW app → Anonymous statistics: the one switch for both daily +// reports, and the messages this box adds to FTW's fleet and energy totals. // // This tab is not a privacy policy. A policy is a promise; this screen shows // the actual payload, built by the same call the sender uses, so somebody @@ -79,16 +80,35 @@ // clock, and until then one box's arrival times are still loosely related. function whereLine(body) { var endpoint = (body && body.endpoint) || "nowhere"; - var handling = body && body.ftw_relay - ? " FTW Cloud adds the report to that day's totals, then drops it and does not save the source address. These are reports, not unique boxes." - : " This box uses a custom collector. FTW cannot say what that service stores or whether it counts unique boxes."; + var relayed = !!(body && body.through_relay); + var where = relayed ? "through " + endpoint + ", an independent relay, to FTW Cloud" : "to " + endpoint; + var handling = relayed + ? " The relay sees this box's address but not the report. FTW Cloud reads the report but not the address, adds it to that day's totals and keeps nothing of it. These are reports, not unique boxes." + : body && body.ftw_relay + ? " FTW Cloud adds the report to that day's totals, then drops it and does not save the source address. These are reports, not unique boxes." + : " This box uses a custom collector. FTW cannot say what that service stores or whether it counts unique boxes."; + if (body && body.waits_for_relay) { + return "Not sent yet. This box shares statistics without having said yes, so this report goes to FTW Cloud only through an independent relay, which hides this box's address, and this release names none yet."; + } if (body && body.enabled) { - return "Sent to " + endpoint + + return "Sent " + where + ", once a day, at a time that moves from day to day — it takes a few days to work its way round the clock." + handling; } - return "Nothing is sent. This is the message that would go to " + endpoint + - ", once a day, if you switched it on." + handling; + return "Nothing is sent. This is the message that would go " + where + + ", once a day, if statistics were on." + handling; + } + + // shareLine says what the one statistics switch means right now. With FTW + // Cloud it is part of Cloud; without, it is the owner's to turn off. + function shareLine(status) { + if (!status) return "Could not read this box's statistics answer."; + if (status.statisticsRequired) { + return "Part of FTW Cloud, with no separate off switch. To stop, turn FTW Cloud off, then turn statistics off here."; + } + return status.statistics + ? "On. Turning it off takes effect at once, with no restart, and the box keeps the answer." + : "Off. Nothing is sent. Turning it on takes effect at once."; } // Labels for the daily energy report. As above, a field with no label is @@ -207,6 +227,41 @@ slot.appendChild(exact); } + var sharePolicy = null; + + // refreshShare reads the box's one statistics answer into the switch. + function refreshShare() { + var box = document.getElementById("statistics-share"); + var note = document.getElementById("statistics-share-note"); + if (!box || !note) return; + Promise.all([fetch("/api/cloud"), fetch("/api/cloud/stats-policy")]) + .then(function (rs) { return Promise.all(rs.map(function (r) { return r.ok ? r.json() : null; })); }) + .then(function (v) { + sharePolicy = v[1]; + box.checked = !!(v[0] && v[0].statistics); + box.disabled = !v[0] || !v[1] || !!v[0].statisticsRequired; + note.textContent = shareLine(v[0]); + }) + .catch(function () { note.textContent = shareLine(null); }); + } + + // The switch writes at once, as the Cloud terms do, and not on Save: it is + // the box's answer, not part of the config document. + function changeShare(box) { + var note = document.getElementById("statistics-share-note"); + box.disabled = true; + var write = box.checked + ? fetch("/api/cloud/stats", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ policy: sharePolicy && sharePolicy.policy }) }) + : fetch("/api/cloud/stats", { method: "DELETE" }); + write + .then(function (r) { return r.json().then(function (v) { if (!r.ok) throw Error(v.error || "The box refused the change."); }); }) + .then(function () { refreshShare(); refresh(); }) + .catch(function (e) { + refreshShare(); + if (note) note.textContent = e.message || "The change could not be saved. Try again."; + }); + } + function refresh() { var slot = document.getElementById("fleet-ping-payload"); if (!slot) return; @@ -233,31 +288,34 @@ }); } - // The FTW app and fleet statistics share a settings page, but not a - // setting: a box can share its anonymous daily report without using the - // app relay. Keep this as a reusable panel instead of a tab so the payload - // and privacy wording stay in one place. + // The FTW app and statistics share a settings page, but not a setting: a + // box shares its anonymous reports whether or not it uses the app. Keep + // this as a reusable panel instead of a tab so the payloads and privacy + // wording stay in one place. S.fleetPing = { - _pure: { payloadRows: payloadRows, describe: describe, whereLine: whereLine, energyRows: energyRows, energyLine: energyLine }, - - // The line under the payload is the box's answer, not the checkbox's, so - // it can only be right again once the box has the new config. Saving is - // the moment that happens. - afterSave: refresh, + _pure: { payloadRows: payloadRows, describe: describe, whereLine: whereLine, shareLine: shareLine, energyRows: energyRows, energyLine: energyLine }, - render: function (ctx) { - // Mirrors the box's own default (see applyDefaults in - // go/internal/config): a box that has never had the section is sending. - // Rendering it unchecked here would say the opposite of what is true. - if (!ctx.config.fleet_ping) ctx.config.fleet_ping = { enabled: true }; - var enabled = !!ctx.config.fleet_ping.enabled; + // Saving the config does not change the answer, but the payload under it + // is built from the config, so read both again. + afterSave: function () { refresh(); refreshShare(); }, - setTimeout(refresh, 0); + render: function () { + setTimeout(function () { + var box = document.getElementById("statistics-share"); + if (box) box.addEventListener("change", function () { changeShare(box); }); + refreshShare(); + refresh(); + }, 0); return ( - "
Anonymous daily report" + - '

The project statistics choice above controls whether this box contributes. This additional daily report describes versions and integrations. It does not count unique boxes.

' + - '

When sharing is on, this report carries six things once a day: its ' + + "

Anonymous statistics" + + '

This box sends two anonymous reports a day, each on its own and at its own time: ' + + "a fleet report and an energy report. With FTW Cloud they are part of Cloud. Without Cloud they " + + "are on until you turn them off here.

" + + '' + + '

Checking…

' + + '

Daily fleet report

' + + '

The fleet report carries six things once a day: its ' + "FTW version, its release channel, the kinds of device it talks to, " + "roughly how much battery, its price zone, and roughly how old the " + "install is. Nothing else — no id, no key, no serial, no site name, no " + @@ -265,22 +323,20 @@ "it. The message itself cannot prove how many unique boxes sent reports.

" + '

Two limits, plainly. Those six things still describe ' + "you: a beta box in a small price zone with a big battery may be the " + - "only one of its kind. And the receiving service sees the address during " + + "only one of its kind. And without a relay the receiving service sees the address during " + "the request, the way any website does. What it keeps depends on the " + "endpoint shown below. Here is the whole message, as this box would send it right now.

" + - '" + - '

Takes effect as soon as you save. No restart.

' + '
' + '

Daily energy report

' + - '

The same switch sends one more report a day: the date, and the ' + + '

The other report: the date, and the ' + "day's solar generation, grid import and export, battery charge and " + "discharge, and vehicle charge, each in whole kWh. It carries no id and " + - "nothing from the report above, so the two cannot be matched. It goes " + - "through an independent relay, so FTW Cloud reads the numbers without " + - "this box's address, and the relay sees the address without the numbers. " + - "A day with a gap in the energy record is skipped.

" + + "nothing from the fleet report, and leaves at its own time, so the two " + + "cannot be matched. They never travel together: device kinds and a price " + + "zone can point at one home, and the energy beside them would then be " + + "that home's. It goes through an independent relay, so FTW Cloud reads " + + "the numbers without this box's address, and the relay sees the address " + + "without the numbers. A day with a gap in the energy record is skipped.

" + '
' + "
" ); From 0ede230d50683e3cb46629a3b38253929e422536 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:04:45 +0200 Subject: [PATCH 3/8] docs(adr): a public energy sum needs five reports that had any of it Co-Authored-By: Claude Opus 5.5 --- docs/adr/0008-ftw-cloud.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index 9f4843cfd..0a8e73f26 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -419,7 +419,9 @@ ciphertext of fixed length, never a report. Cloudflare's relay, if chosen, keeps those logs for about 124 days. The gateway sees each report, the relay's address and the time, never the box's address. FTW Cloud keeps per date only the counts and sums; a public energy date appears once no report -may still name it, four days after it ends, and needs at least 5 reports. +may still name it, four days after it ends, and needs at least 5 reports. A +sum that one to four of them had any of shows null: five reports with one car +would otherwise publish that car's charge. Limits, plainly: From a7c025e8722117e8ac1318414990099dcfa4bfbb Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:08:11 +0200 Subject: [PATCH 4/8] fix(statsrelay): each report carries an id and the day it was sent From review of srcfl/ftw-cloud#50: the gateway counted a replayed request again, and a fleet report, which has no date, could be replayed on any later day. Each inner request now carries a random report id and the UTC date it was sent, as Binary HTTP header fields inside the sealed message, at the same fixed length. reportwire.Fresh refuses a report sent before yesterday; the gateway keeps an id hash per date to count each report once. ADR 0008 also states the open limit: the floor of five counts reports, and anyone with the key configuration can post them. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0008-ftw-cloud.md | 18 +++-- go/internal/statsrelay/statsrelay.go | 14 +++- .../statsrelaytest/statsrelaytest.go | 5 +- go/protocol/reportwire/report.go | 66 +++++++++++++++---- go/protocol/reportwire/report_test.go | 45 ++++++++++--- 5 files changed, 119 insertions(+), 29 deletions(-) diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index 0a8e73f26..c62e91380 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -400,8 +400,13 @@ together. **Path.** Both go through one independent Oblivious HTTP relay (RFC 9458) to FTW Cloud's gateway, `POST /v1/ohttp` in `ftwcloudd`, as one padded inner request each (`go/protocol/reportwire`, 1,536 bytes; the relay always sees -1,591). The gateway reads the schema, adds the report to its own day totals -(`/v1/fleet/public`, `/v1/energy/public`) and keeps nothing per report. +1,591). Inside, each report carries a random id and the UTC date it was +sent. The gateway reads the schema, adds the report to its own day totals +(`/v1/fleet/public`, `/v1/energy/public`) and keeps nothing per report but +a hash of its id, for three days, so a replay or a retry counts once +(RFC 9458, section 6.5). A report sent before yesterday is refused. Both +public views list closed days only, so nobody can match a new count to the +report that just arrived. `go/internal/statsrelay` holds the relay URL and the gateway key as compile-time pins. The box encrypts only to the pinned key and never asks the service for one, so every box on a release uses the same key and a key @@ -432,9 +437,12 @@ Limits, plainly: computed from them and matched with a report. The gateway does not do this and keeps nothing to match with, but that is a rule in code, not one the cryptography enforces. -- An anonymous report cannot prove that a real box sent it, and the relay - could replay one. Bounds per field and a cap per date limit junk to noise. - Privacy Pass-style tokens are the later fix; they are not built. +- An anonymous report cannot prove that a real box sent it. Anyone with the + public key configuration can post reports, and the relay can drop some. + One real report and four all-zero fakes would publish that home's six + sums under "5 reports". The floor of five counts reports, not boxes; this + is open, for Fredrik. Privacy Pass-style tokens would close it; they are + not built. **Open for Fredrik's lawyer, not decided here.** Reports that are anonymous to Sourceful through OHTTP should be outside GDPR. Reading data from the diff --git a/go/internal/statsrelay/statsrelay.go b/go/internal/statsrelay/statsrelay.go index 5112224a4..d5f260b74 100644 --- a/go/internal/statsrelay/statsrelay.go +++ b/go/internal/statsrelay/statsrelay.go @@ -12,6 +12,7 @@ package statsrelay import ( "bytes" "context" + "crypto/rand" "encoding/base64" "errors" "fmt" @@ -52,6 +53,7 @@ type Client struct { relay string key ohttp.KeyConfig http *http.Client + now func() time.Time } // Pinned is this build's client, or nil when the build names no usable relay @@ -90,7 +92,7 @@ func New(relay string, key ohttp.KeyConfig, client *http.Client) *Client { } // A redirect would resend the sealed report to an address nobody chose. client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } - return &Client{relay: relay, key: key, http: client} + return &Client{relay: relay, key: key, http: client, now: time.Now} } // Relay is the relay's URL, for Settings. @@ -98,8 +100,16 @@ func (c *Client) Relay() string { return c.relay } // Send seals one report body and reads the gateway's answer. Only a sealed // 204 from the gateway the box encrypted to counts as delivered. +// +// Each call is one report: it draws a fresh id, which the gateway counts +// once. Nothing retries; a retry of the same report would have to reuse its +// id, or the gateway would count it twice. func (c *Client) Send(ctx context.Context, body []byte) error { - inner, ok := reportwire.Request(body) + id := make([]byte, reportwire.IDBytes) + if _, err := rand.Read(id); err != nil { + return err + } + inner, ok := reportwire.Request(reportwire.Report{ID: id, Day: c.now().UTC().Format(time.DateOnly), Body: body}) if !ok { return errors.New("statsrelay: the report does not fit the padded request") } diff --git a/go/internal/statsrelay/statsrelaytest/statsrelaytest.go b/go/internal/statsrelay/statsrelaytest/statsrelaytest.go index 1d6a66b1b..1a30e5bfa 100644 --- a/go/internal/statsrelay/statsrelaytest/statsrelaytest.go +++ b/go/internal/statsrelay/statsrelaytest/statsrelaytest.go @@ -93,10 +93,11 @@ func (g *Gateway) add(msg []byte) int { g.mu.Lock() defer g.mu.Unlock() g.last = http.StatusBadRequest - body, ok := reportwire.Body(msg) - if !ok { + report, ok := reportwire.Read(msg) + if !ok || !reportwire.Fresh(report.Day, g.now) { return g.last } + body := report.Body switch reportwire.Schema(body) { case fleetwire.Schema: if r, err := fleetwire.Decode(body); err == nil { diff --git a/go/protocol/reportwire/report.go b/go/protocol/reportwire/report.go index efafc0cd4..a69557be4 100644 --- a/go/protocol/reportwire/report.go +++ b/go/protocol/reportwire/report.go @@ -10,7 +10,9 @@ package reportwire import ( + "encoding/base64" "encoding/json" + "time" "github.com/srcfl/ftw/go/protocol/ohttp" ) @@ -31,23 +33,65 @@ const ( ResponseSize = 16 ) -// Request is the padded inner request carrying one report body. ok is false -// when the body does not fit, so a report never goes out at a length of its -// own. -func Request(body []byte) (msg []byte, ok bool) { - msg = ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: body}.Marshal(RequestSize) +// IDBytes is a report id's length. The box draws one at random for each +// report and keeps it if it ever sends that report again, so the gateway +// counts a report once however often it arrives (RFC 9458, section 6.5). +const IDBytes = 16 + +// Report is what one inner request carries besides its fixed parts: the +// report's random id, the UTC date the box sent it, and the report itself. +// The id names one report, never a box. +type Report struct { + ID []byte + Day string + Body []byte +} + +const ( + idField = "report-id" + dayField = "report-day" +) + +// Request is the padded inner request carrying one report. ok is false for +// an id or a day of the wrong form, and when the report does not fit, so a +// report never goes out at a length of its own. +func Request(r Report) (msg []byte, ok bool) { + if len(r.ID) != IDBytes || !isDate(r.Day) { + return nil, false + } + msg = ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, + Fields: []ohttp.Field{{Name: idField, Value: base64.RawURLEncoding.EncodeToString(r.ID)}, {Name: dayField, Value: r.Day}}, + Content: r.Body}.Marshal(RequestSize) return msg, len(msg) == RequestSize } -// Body opens an inner request and returns the report it carries, or false -// for anything but the one padded request. -func Body(msg []byte) ([]byte, bool) { +// Read opens an inner request, or returns false for anything but the one +// padded request with an id and a day. +func Read(msg []byte) (Report, bool) { req, err := ohttp.ParseRequest(msg) if err != nil || len(msg) != RequestSize || req.Method != Method || req.Scheme != Scheme || - req.Authority != Authority || req.Path != Path || len(req.Fields) != 0 { - return nil, false + req.Authority != Authority || req.Path != Path || len(req.Fields) != 2 || + req.Fields[0].Name != idField || req.Fields[1].Name != dayField { + return Report{}, false } - return req.Content, true + id, err := base64.RawURLEncoding.DecodeString(req.Fields[0].Value) + if err != nil || len(id) != IDBytes || !isDate(req.Fields[1].Value) { + return Report{}, false + } + return Report{ID: id, Day: req.Fields[1].Value, Body: req.Content}, true +} + +// Fresh reports whether a report sent on day still counts at now: sent today +// or yesterday, UTC. An older one is refused, so a captured request cannot be +// posted again on a later day, when its id is no longer remembered. +func Fresh(day string, now time.Time) bool { + today := now.UTC().Format(time.DateOnly) + return day == today || day == now.UTC().AddDate(0, 0, -1).Format(time.DateOnly) +} + +func isDate(s string) bool { + t, err := time.Parse(time.DateOnly, s) + return err == nil && t.Format(time.DateOnly) == s } // Schema reads only a report's schema, to choose its decoder. The decoder diff --git a/go/protocol/reportwire/report_test.go b/go/protocol/reportwire/report_test.go index a4c63f8bd..7afe0ec8e 100644 --- a/go/protocol/reportwire/report_test.go +++ b/go/protocol/reportwire/report_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" "testing" + "time" "github.com/srcfl/ftw/go/protocol/energywire" "github.com/srcfl/ftw/go/protocol/fleetwire" @@ -23,21 +24,29 @@ func TestEveryReportHasOneLength(t *testing.T) { energy, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", PVGeneration: 1000, GridImport: 1000, GridExport: 1000, BatteryCharge: 1000, BatteryDischarge: 1000, VehicleCharge: 1000}) small, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23"}) + id := []byte("0123456789abcdef") for _, body := range [][]byte{fleet, energy, small} { - msg, ok := Request(body) + msg, ok := Request(Report{ID: id, Day: "2026-10-24", Body: body}) if !ok || len(msg) != RequestSize { t.Fatalf("%d-byte report padded to %d", len(body), len(msg)) } - got, ok := Body(msg) - if !ok || string(got) != string(body) || Schema(got) == "" { - t.Fatalf("read back %q", got) + got, ok := Read(msg) + if !ok || string(got.Body) != string(body) || string(got.ID) != string(id) || got.Day != "2026-10-24" || Schema(got.Body) == "" { + t.Fatalf("read back %+v", got) } } if Schema(fleet) != fleetwire.Schema || Schema(energy) != energywire.Schema || Schema([]byte("x")) != "" { t.Fatal("schema") } - if _, ok := Request(make([]byte, RequestSize)); ok { - t.Fatal("an oversized report was padded") + for name, r := range map[string]Report{ + "oversized": {ID: id, Day: "2026-10-24", Body: make([]byte, RequestSize)}, + "short id": {ID: id[:15], Day: "2026-10-24", Body: small}, + "no day": {ID: id, Body: small}, + "loose day": {ID: id, Day: "2026-10-4", Body: small}, + } { + if _, ok := Request(r); ok { + t.Fatalf("%s was padded", name) + } } if len(Response(204)) != ResponseSize { t.Fatal("response padding") @@ -45,7 +54,11 @@ func TestEveryReportHasOneLength(t *testing.T) { } func TestOnlyTheOneInnerRequestIsRead(t *testing.T) { - good := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Content: []byte("{}")} + fields := []ohttp.Field{{Name: "report-id", Value: "MDEyMzQ1Njc4OWFiY2RlZg"}, {Name: "report-day", Value: "2026-10-24"}} + good := ohttp.Request{Method: Method, Scheme: Scheme, Authority: Authority, Path: Path, Fields: fields, Content: []byte("{}")} + if _, ok := Read(good.Marshal(RequestSize)); !ok { + t.Fatal("the one request was refused") + } other := func(f func(*ohttp.Request)) []byte { r := good f(&r) @@ -56,13 +69,27 @@ func TestOnlyTheOneInnerRequestIsRead(t *testing.T) { "another method": other(func(r *ohttp.Request) { r.Method = "PUT" }), "another host": other(func(r *ohttp.Request) { r.Authority = "example.com" }), "plain http": other(func(r *ohttp.Request) { r.Scheme = "http" }), - "a header": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "x-home", Value: "1"}} }), + "another header": other(func(r *ohttp.Request) { r.Fields = append(r.Fields, ohttp.Field{Name: "x-home", Value: "1"}) }), + "no id": other(func(r *ohttp.Request) { r.Fields = r.Fields[1:] }), + "short id": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "report-id", Value: "MDEy"}, fields[1]} }), + "bad day": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{fields[0], {Name: "report-day", Value: "yesterday"}} }), + "swapped": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{fields[1], fields[0]} }), "not padded": good.Marshal(0), "padded more": good.Marshal(RequestSize + 1), "a response": Response(200), } { - if _, ok := Body(msg); ok { + if _, ok := Read(msg); ok { t.Fatalf("%s was read", name) } } } + +// A report counts on the day it was sent and the next, UTC, and no later. +func TestOnlyAFreshReportCounts(t *testing.T) { + now := time.Date(2026, 10, 24, 0, 30, 0, 0, time.UTC) + for day, want := range map[string]bool{"2026-10-24": true, "2026-10-23": true, "2026-10-22": false, "2026-10-25": false, "x": false} { + if Fresh(day, now) != want { + t.Fatalf("%s: fresh %v", day, !want) + } + } +} From 1ecd67f1cc2c2533b9dcdaac62e1b5a2eef1caab Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:28:42 +0200 Subject: [PATCH 5/8] fix(stats): the hourly upload follows the one answer; whole days with new sessions From review of #1820: - The hourly C8 upload read only the grant's stats purpose. It now reads the one answer too: off, or Cloud under older terms, sends no hour. Under older terms, asking for statistics is refused (ErrCloudTermsFirst, 409) instead of adding the stats purpose beside the old terms; Status says so (statisticsNeedTerms) and Settings shows it. After the migration, ReconcileStatistics takes the stats purpose out of a grant whose answer is off, and ends a statistics-only link, as turning statistics off does. - A stored grant counts only through the running link, which honours it only with its key (Link.SharesStatistics, Link.SaidYesToStatistics). With no link, the box's own answer alone. - While a Cloud that was turned off waits for its deletion receipt, the local answer can still change. - energyreport.Day: a charger's session counter starts again at every plug-in, which the ledger marks as a reset and covers from power; that no longer drops the day. An asset first seen after the day ended is not part of it. - statsrelay: FTW Cloud's host is refused in any case and with a trailing dot. - ADR 0008 records Fredrik's decisions: a default box waits for the relay, and the floor of five counts reports, accepted, with Privacy Pass later. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0008-ftw-cloud.md | 29 ++++-- go/cmd/ftw/energy_report.go | 9 +- go/cmd/ftw/energy_report_test.go | 23 +++-- go/cmd/ftw/fleet_ping.go | 58 ++++++++--- go/cmd/ftw/fleet_ping_test.go | 6 +- go/cmd/ftw/main.go | 7 +- go/internal/api/api_cloud.go | 2 +- go/internal/api/api_cloud_test.go | 20 ++-- go/internal/api/api_fleet_ping.go | 7 +- go/internal/cloudlink/choice.go | 89 ++++++++++++---- go/internal/cloudlink/choice_test.go | 31 ++---- go/internal/cloudlink/link.go | 14 ++- go/internal/cloudlink/link_test.go | 117 ++++++++++----------- go/internal/cloudlink/stats.go | 5 + go/internal/cloudlink/stats_test.go | 120 +++++++++++++++++----- go/internal/energyreport/day.go | 14 +-- go/internal/energyreport/day_test.go | 41 ++++++++ go/internal/statsrelay/statsrelay.go | 9 +- go/internal/statsrelay/statsrelay_test.go | 2 + web/fleet-ping-tab.test.mjs | 6 ++ web/settings/tabs/fleet.js | 5 +- 21 files changed, 418 insertions(+), 196 deletions(-) diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index c62e91380..9a5ba5274 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -381,6 +381,12 @@ anonymous daily reports, with no second setting: already running keep working until turned off. Hourly C8 rows for local boxes therefore have no way in; if C8 is ever activated for them, it needs its own consent. +- The hourly C8 upload follows the same answer: off, or Cloud under older + terms, sends no hour whatever the grant carries, and an answer that is off + takes the stats purpose out of the grant. Under older terms, asking for + statistics is refused until the owner accepts the current terms. +- A stored grant counts only through the running link, which honours it only + with the key it was made with. **Two reports, never one message.** Each leaves at its own random time: @@ -414,10 +420,11 @@ swapped in at the service opens nothing. Until both pins are set, the energy report is not sent, and the fleet report posts straight to `/v1/fleet` as before, but only from a box that said yes: FTW Cloud's terms or a local yes. A box that shares only by default never posts straight to FTW Cloud, which -would see its address; it waits for the relay. A build with the relay sends -the fleet report to FTW Cloud only through it. `POST /v1/fleet` stays for boxes on older releases; it can go one -stable release after the first that pins the relay. A collector the household -set in `fleet_ping.endpoint` is still posted to directly. +would see its address; it waits for the relay (Fredrik, 24 October 2026). A +build with the relay sends the fleet report to FTW Cloud only through it. +`POST /v1/fleet` stays for boxes on older releases; it can go one stable +release after the first that pins the relay. A collector the household set +in `fleet_ping.endpoint` is still posted to directly. **Who sees what.** The relay sees the box's address, the time and a ciphertext of fixed length, never a report. Cloudflare's relay, if chosen, @@ -437,12 +444,14 @@ Limits, plainly: computed from them and matched with a report. The gateway does not do this and keeps nothing to match with, but that is a rule in code, not one the cryptography enforces. -- An anonymous report cannot prove that a real box sent it. Anyone with the - public key configuration can post reports, and the relay can drop some. - One real report and four all-zero fakes would publish that home's six - sums under "5 reports". The floor of five counts reports, not boxes; this - is open, for Fredrik. Privacy Pass-style tokens would close it; they are - not built. +- The floor of five counts reports, not boxes; Fredrik accepted this on + 24 October 2026. An anonymous report cannot prove that a real box sent + it. The relay alone can drop the other reports and add fakes to isolate + one home's day, and that shows on our side as missing reports. Anyone + else can do it only on a day when one real home reported. Rejecting + replays and listing closed days only remove the cheap attacks. Anonymous + tokens (Privacy Pass) are the later fix, once the fleet grows or someone + shows a need. **Open for Fredrik's lawyer, not decided here.** Reports that are anonymous to Sourceful through OHTTP should be outside GDPR. Reading data from the diff --git a/go/cmd/ftw/energy_report.go b/go/cmd/ftw/energy_report.go index 276c2930c..e40d8b84b 100644 --- a/go/cmd/ftw/energy_report.go +++ b/go/cmd/ftw/energy_report.go @@ -18,12 +18,12 @@ import ( // startEnergyReport builds the sender and starts its schedule. It is built // even when nothing may be sent, so Settings can show the report this box // would send. A build with no relay or gateway key sends nothing at all. -func startEnergyReport(ctx context.Context, cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) *energyreport.Sender { +func startEnergyReport(ctx context.Context, cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store, answer statsAnswer) *energyreport.Sender { if st == nil { return nil } sender, err := energyreport.New(energyreport.Options{ - Enabled: energyReportAllowed(cfg, cfgMu, st), + Enabled: energyReportAllowed(cfg, cfgMu, answer), Ledger: st, KV: st, ClockSynced: cloudlink.ClockSynced, @@ -39,12 +39,11 @@ func startEnergyReport(ctx context.Context, cfg *config.Config, cfgMu *sync.RWMu // energyReportAllowed is the fleet report's answer, and no site a partner // controls: as for the hourly project statistics, a partner's contract has to // let such a home in first. -func energyReportAllowed(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) func() bool { - allowed := sharesStatistics(st) +func energyReportAllowed(cfg *config.Config, cfgMu *sync.RWMutex, answer statsAnswer) func() bool { return func() bool { cfgMu.RLock() partner := cfg.Nova != nil && cfg.Nova.Enabled cfgMu.RUnlock() - return !partner && allowed() + return !partner && answer.shares() } } diff --git a/go/cmd/ftw/energy_report_test.go b/go/cmd/ftw/energy_report_test.go index e8b9e9500..f2423a58f 100644 --- a/go/cmd/ftw/energy_report_test.go +++ b/go/cmd/ftw/energy_report_test.go @@ -26,7 +26,8 @@ func TestBothReportsReadTheOneStatisticsAnswer(t *testing.T) { t.Cleanup(func() { _ = st.Close() }) cfg := &config.Config{FleetPing: &config.FleetPing{Enabled: true}} cfgMu := &sync.RWMutex{} - energy, fleet := energyReportAllowed(cfg, cfgMu, st), sharesStatistics(st) + answer := statsAnswer{st: st} + energy, fleet := energyReportAllowed(cfg, cfgMu, answer), answer.shares check := func(name string, wantEnergy, wantFleet bool) { t.Helper() if energy() != wantEnergy || fleet() != wantFleet { @@ -35,7 +36,7 @@ func TestBothReportsReadTheOneStatisticsAnswer(t *testing.T) { } check("before the stored state is read", false, false) - migrateStatistics(cfg, cfgMu, st) + migrateStatistics(cfg, cfgMu, st, nil) check("a box without Cloud, never asked", true, true) // The old switch is read once and is no longer a second setting. @@ -51,11 +52,14 @@ func TestBothReportsReadTheOneStatisticsAnswer(t *testing.T) { } check("the owner turned statistics off", false, false) + // A stored Cloud grant counts only through a running link, which honours + // it only with its key (cloudlink's tests). Without one, the box's own + // answer stands. grant, _ := json.Marshal(map[string]any{"policy": cloudwire.CloudTermsPolicy, "purposes": []string{"serve", "stats"}}) if err := st.SaveConfig("cloud/grant_v1", string(grant)); err != nil { t.Fatal(err) } - check("FTW Cloud under the current terms", true, true) + check("a stored grant with no link", false, false) } // A box that shares only by default posts nothing straight to FTW Cloud: its @@ -67,21 +71,22 @@ func TestADefaultBoxSendsNothingStraightToFTWCloud(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) - migrateStatistics(&config.Config{}, &sync.RWMutex{}, st) + migrateStatistics(&config.Config{}, &sync.RWMutex{}, st, nil) + answer := statsAnswer{st: st} client, _, _ := statsrelaytest.New(t, time.Now()) direct, _ := fleetProvider(config.DefaultFleetPingEndpoint, nil) relayed, _ := fleetProvider(config.DefaultFleetPingEndpoint, client) own, _ := fleetProvider("https://collector.example.test/fleet", nil) - if fleetReportAllowed(st, config.DefaultFleetPingEndpoint, direct)() { + if fleetReportAllowed(answer, config.DefaultFleetPingEndpoint, direct)() { t.Fatal("a box that never said yes posted straight to FTW Cloud") } - if !fleetReportAllowed(st, config.DefaultFleetPingEndpoint, relayed)() || !fleetReportAllowed(st, "https://collector.example.test/fleet", own)() { + if !fleetReportAllowed(answer, config.DefaultFleetPingEndpoint, relayed)() || !fleetReportAllowed(answer, "https://collector.example.test/fleet", own)() { t.Fatal("the relay or the household's collector was held back") } if err := st.SaveConfig("statistics/share_v1", "on"); err != nil { t.Fatal(err) } - if !fleetReportAllowed(st, config.DefaultFleetPingEndpoint, direct)() { + if !fleetReportAllowed(answer, config.DefaultFleetPingEndpoint, direct)() { t.Fatal("a yes did not send") } } @@ -94,8 +99,8 @@ func TestASwitchedOffFleetReportStaysOff(t *testing.T) { } t.Cleanup(func() { _ = st.Close() }) cfg := &config.Config{FleetPing: &config.FleetPing{Enabled: false}} - migrateStatistics(cfg, &sync.RWMutex{}, st) - if sharesStatistics(st)() { + migrateStatistics(cfg, &sync.RWMutex{}, st, nil) + if (statsAnswer{st: st}).shares() { t.Fatal("an earlier off was turned on") } } diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go index 970d3e544..c64badd9d 100644 --- a/go/cmd/ftw/fleet_ping.go +++ b/go/cmd/ftw/fleet_ping.go @@ -64,6 +64,7 @@ func startFleetPing( cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store, + answer statsAnswer, version string, managedDriverDir string, ) (*fleetping.Pinger, error) { @@ -79,7 +80,7 @@ func startFleetPing( installedAt := firstSeen(st, time.Now()) pinger, err := fleetping.New(fleetping.Options{ - Enabled: fleetReportAllowed(st, endpoint, provider), + Enabled: fleetReportAllowed(answer, endpoint, provider), Facts: func() fleetping.Facts { // Drivers can be installed without restarting FTW. Read the signed // catalogue at the same time as the config so tomorrow's report and @@ -111,36 +112,61 @@ func fleetProvider(endpoint string, relay *statsrelay.Client) (fleetping.Provide return fleetping.NewHTTPProvider(endpoint, nil) } -// sharesStatistics is the one answer for both anonymous daily reports, the -// fleet report and the energy report: part of FTW Cloud, and on for a box -// without it until the owner turns it off. One function, so the two reports -// cannot come to read different answers. fleet_ping.enabled is no longer a -// second switch; MigrateStatistics read it once. -func sharesStatistics(st *state.Store) func() bool { - return func() bool { return st != nil && cloudlink.SharesStatistics(st) } +// statsAnswer is the one answer for both anonymous daily reports, the fleet +// report and the energy report: part of FTW Cloud, and on for a box without +// it until the owner turns it off. One reader, so the two reports cannot come +// to read different answers. It asks the running Cloud link, which counts a +// grant only with the key it was made with; with no link, the box's own +// answer alone. fleet_ping.enabled is no longer a second switch; +// migrateStatistics read it once. +type statsAnswer struct { + st *state.Store + link *cloudlink.Link +} + +func (a statsAnswer) shares() bool { + if a.link != nil { + return a.link.SharesStatistics() + } + return a.st != nil && cloudlink.SharesStatistics(a.st) +} + +func (a statsAnswer) saidYes() bool { + if a.link != nil { + return a.link.SaidYesToStatistics() + } + return a.st != nil && cloudlink.SaidYesToStatistics(a.st) } // fleetReportAllowed is the one answer, with one more rule for the direct -// path: a box that shares only by default, never having said yes, posts -// nothing straight to FTW Cloud, which would see its address. Through the -// relay, or to a collector the household chose, the answer alone decides. -func fleetReportAllowed(st *state.Store, endpoint string, provider fleetping.Provider) func() bool { - shares := sharesStatistics(st) +// path (Fredrik, 24 October 2026): a box that shares only by default, never +// having said yes, posts nothing straight to FTW Cloud, which would see its +// address; it waits for the relay. Through the relay, or to a collector the +// household chose, the answer alone decides. +func fleetReportAllowed(answer statsAnswer, endpoint string, provider fleetping.Provider) func() bool { _, relayed := provider.(*fleetping.RelayProvider) direct := !relayed && strings.TrimSpace(endpoint) == config.DefaultFleetPingEndpoint return func() bool { - return shares() && (!direct || cloudlink.SaidYesToStatistics(st)) + return answer.shares() && (!direct || answer.saidYes()) } } // migrateStatistics turns what an earlier release stored into the one -// answer, once: a fleet report switched off in the config stays off. -func migrateStatistics(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store) { +// answer, once: a fleet report switched off in the config stays off. Then +// the Cloud grant follows it, so an answer that came out off also stops an +// hourly upload the grant still carried. +func migrateStatistics(cfg *config.Config, cfgMu *sync.RWMutex, st *state.Store, link *cloudlink.Link) { if st == nil { return } if err := cloudlink.MigrateStatistics(st, !fleetPingEnabled(cfg, cfgMu)()); err != nil { slog.Warn("could not record the statistics answer; nothing is sent until it is", "err", err) + return + } + if link != nil { + if err := link.ReconcileStatistics("statistics answer"); err != nil { + slog.Warn("could not make the FTW Cloud grant follow the statistics answer", "err", err) + } } } diff --git a/go/cmd/ftw/fleet_ping_test.go b/go/cmd/ftw/fleet_ping_test.go index 142136992..f8c323bf2 100644 --- a/go/cmd/ftw/fleet_ping_test.go +++ b/go/cmd/ftw/fleet_ping_test.go @@ -117,7 +117,7 @@ func TestRunningFleetPingSeesADriverInstalledLater(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } @@ -339,7 +339,7 @@ func TestNoNameTheHouseholdChoseReachesThePostedPing(t *testing.T) { // path unchanged. Payload() is what sendOnce marshals and posts. ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } @@ -469,7 +469,7 @@ func TestARepositoryRootInsideTheBundledTreeIsNotTheShippedCatalogue(t *testing. ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 88e80ef2f..e77754b41 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2492,16 +2492,17 @@ func main() { // The bundled drivers dir is deliberately not passed: which drivers this // build ships with is compiled in, not discovered on disk. See // fleetCatalogue for what that buys. - migrateStatistics(cfg, cfgMu, st) + migrateStatistics(cfg, cfgMu, st, cloudLink) + answer := statsAnswer{st: st, link: cloudLink} fleetPinger, fleetPingErr := startFleetPing( - ctx, cfg, cfgMu, st, Version, config.ManagedDriversDirOverride, + ctx, cfg, cfgMu, st, answer, Version, config.ManagedDriversDirOverride, ) if fleetPingErr != nil { slog.Warn("fleet ping unavailable", "err", fleetPingErr) } // The anonymous daily energy report, under the same answer, through an // independent relay so that FTW Cloud never learns which box sent it. - energyReport := startEnergyReport(ctx, cfg, cfgMu, st) + energyReport := startEnergyReport(ctx, cfg, cfgMu, st, answer) mutationPolicy := apiMutationPolicy() mutationPolicy.LANAuthEnabled = func() bool { diff --git a/go/internal/api/api_cloud.go b/go/internal/api/api_cloud.go index 331d9a3bc..2d7b86126 100644 --- a/go/internal/api/api_cloud.go +++ b/go/internal/api/api_cloud.go @@ -189,7 +189,7 @@ func (s *Server) handleCloudStats(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, link.Status()) case errors.Is(err, cloudlink.ErrGrantRequest): writeAppLinkError(w, http.StatusBadRequest, err.Error()) - case errors.Is(err, cloudlink.ErrNotGranted), errors.Is(err, cloudlink.ErrPurgePending): + case errors.Is(err, cloudlink.ErrNotGranted), errors.Is(err, cloudlink.ErrPurgePending), errors.Is(err, cloudlink.ErrCloudTermsFirst): writeAppLinkError(w, http.StatusConflict, err.Error()) case errors.Is(err, cloudlink.ErrNoService): writeAppLinkError(w, http.StatusServiceUnavailable, err.Error()) diff --git a/go/internal/api/api_cloud_test.go b/go/internal/api/api_cloud_test.go index e4c064bc1..1f37153de 100644 --- a/go/internal/api/api_cloud_test.go +++ b/go/internal/api/api_cloud_test.go @@ -430,20 +430,28 @@ func TestTheConsentRoundTripShowsThePurposes(t *testing.T) { return st } + current := `{"policy":"` + cloudwire.StatsPolicy + `"}` call(s.handleCloudStats, http.MethodPost, "/api/cloud/stats", statsBody, http.StatusConflict) call(s.handleCloudGrant, http.MethodPost, "/api/cloud/grant", grantBody, http.StatusOK) - call(s.handleCloudStats, http.MethodPost, "/api/cloud/stats", statsBody, http.StatusOK) - if st := call(s.handleCloudStatus, http.MethodGet, "/api/cloud", "", http.StatusOK); st.State != "on" || - strings.Join(st.Purposes, ",") != "serve,stats" { - t.Fatalf("status with statistics on: %+v", st) + // Older terms did not include statistics: the current ones come first. + if st := call(s.handleCloudStatus, http.MethodGet, "/api/cloud", "", http.StatusOK); !st.StatisticsNeedTerms || st.Statistics { + t.Fatalf("status under older terms: %+v", st) } - if st := call(s.handleCloudStopStats, http.MethodDelete, "/api/cloud/stats", "", http.StatusOK); strings.Join(st.Purposes, ",") != "serve" { - t.Fatalf("status with statistics off: %+v", st) + call(s.handleCloudStats, http.MethodPost, "/api/cloud/stats", current, http.StatusConflict) + call(s.handleCloudGrant, http.MethodPost, "/api/cloud/grant", `{"scopes":["ftw.live.read"],"policy":"`+cloudwire.CloudTermsPolicy+`"}`, http.StatusOK) + if st := call(s.handleCloudStatus, http.MethodGet, "/api/cloud", "", http.StatusOK); st.State != "on" || + strings.Join(st.Purposes, ",") != "serve,stats" || !st.Statistics || !st.StatisticsRequired { + t.Fatalf("status under the current terms: %+v", st) } + call(s.handleCloudStopStats, http.MethodDelete, "/api/cloud/stats", "", http.StatusConflict) if st := call(s.handleCloudRevoke, http.MethodDelete, "/api/cloud/grant", "", http.StatusOK); st.State != "revoking" { t.Fatalf("status after FTW Cloud off: %+v", st) } + // While the copy is deleted, the box's own answer can still change. call(s.handleCloudStats, http.MethodPost, "/api/cloud/stats", statsBody, http.StatusConflict) + if st := call(s.handleCloudStats, http.MethodPost, "/api/cloud/stats", current, http.StatusOK); !st.Statistics { + t.Fatalf("status after the local yes: %+v", st) + } } type cloudSiteStub struct{} diff --git a/go/internal/api/api_fleet_ping.go b/go/internal/api/api_fleet_ping.go index 66f9ea191..a07e3fb72 100644 --- a/go/internal/api/api_fleet_ping.go +++ b/go/internal/api/api_fleet_ping.go @@ -58,8 +58,11 @@ func (s *Server) handleFleetPing(w http.ResponseWriter, r *http.Request) { ThroughRelay: s.deps.FleetPing.ThroughRelay(), Payload: s.deps.FleetPing.Payload(), } - view.WaitsForRelay = !enabled && !view.ThroughRelay && endpoint == config.DefaultFleetPingEndpoint && - s.deps.State != nil && cloudlink.SharesStatistics(s.deps.State) + shares := s.deps.State != nil && cloudlink.SharesStatistics(s.deps.State) + if s.deps.Cloud != nil { + shares = s.deps.Cloud.Status().Statistics + } + view.WaitsForRelay = !enabled && !view.ThroughRelay && endpoint == config.DefaultFleetPingEndpoint && shares if s.deps.EnergyReport != nil { p := s.deps.EnergyReport.Preview(r.Context()) view.Energy = &p diff --git a/go/internal/cloudlink/choice.go b/go/internal/cloudlink/choice.go index 07cc677d6..4137df789 100644 --- a/go/internal/cloudlink/choice.go +++ b/go/internal/cloudlink/choice.go @@ -2,12 +2,18 @@ package cloudlink import ( "errors" + "slices" + "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/cloudwire" ) var ErrCloudNeedsStatistics = errors.New("cloudlink: statistics are part of FTW Cloud; turn FTW Cloud off, then statistics, to stop contributing") +// ErrCloudTermsFirst is statistics asked for while Cloud runs under older +// terms, which did not include them. +var ErrCloudTermsFirst = errors.New("cloudlink: accept the current FTW Cloud terms first; they include statistics") + // includeStatistics records the exact scope the owner read. Service approval // still gates every report. The caller holds mu. func (l *Link) includeStatistics(g *grantRecord, fresh bool) error { @@ -78,22 +84,56 @@ func (l *Link) End(by string) error { // With FTW Cloud they are part of Cloud, with no separate off switch. A Cloud // grant under older terms sends nothing until the owner accepts the current // ones. Without Cloud they are on unless the owner turned them off. +// +// This form is for a box whose Cloud link is not running, so no stored grant +// is honoured: the local answer alone. Link.SharesStatistics counts the grant +// the running link honours, which it does only with the key it was made with. func SharesStatistics(kv KV) bool { if kv == nil { return false } - g, err := loadJSON[grantRecord](kv, grantKey) - if err != nil { + share, _ := kv.LoadConfig(statsShareKey) + return sharesStatistics(nil, share) +} + +// SaidYesToStatistics reports an active yes, for a box whose Cloud link is +// not running: a local yes. A box that shares only by default has not given +// one, so its reports reach FTW Cloud only through the relay, which hides its +// address; nothing goes straight to FTW Cloud from it. +func SaidYesToStatistics(kv KV) bool { + if kv == nil { return false } - // Cloud turned off is off at once, before the grant record says so. - if disabled, _ := kv.LoadConfig(disabledKey); disabled == "true" { - g = nil - } share, _ := kv.LoadConfig(statsShareKey) + return saidYesToStatistics(nil, share) +} + +// SharesStatistics is the one answer with the grant this link honours. +func (l *Link) SharesStatistics() bool { + g, share := l.statisticsAnswer() return sharesStatistics(g, share) } +// SaidYesToStatistics is an active yes with the grant this link honours: +// Cloud under the current terms, or a local yes. +func (l *Link) SaidYesToStatistics() bool { + g, share := l.statisticsAnswer() + return saidYesToStatistics(g, share) +} + +// statisticsAnswer is the grant this link honours, nil once Cloud is turned +// off, and the stored local answer. +func (l *Link) statisticsAnswer() (*grantRecord, string) { + l.mu.Lock() + g := l.grant + if l.disabled { + g = nil + } + l.mu.Unlock() + share, _ := l.opts.KV.LoadConfig(statsShareKey) + return g, share +} + func sharesStatistics(g *grantRecord, share string) bool { if g != nil && !g.Revoked && !g.statisticsOnly() { return g.Policy == cloudwire.CloudTermsPolicy @@ -101,28 +141,33 @@ func sharesStatistics(g *grantRecord, share string) bool { return share == "on" || share == "default" } -// SaidYesToStatistics reports an active yes: FTW Cloud under the current -// terms, or a local yes. A box that shares only by default has not given one, -// so its reports reach FTW Cloud only through the relay, which hides its -// address; nothing goes straight to FTW Cloud from it. -func SaidYesToStatistics(kv KV) bool { - if kv == nil { - return false - } - g, err := loadJSON[grantRecord](kv, grantKey) - if err != nil { - return false - } - if disabled, _ := kv.LoadConfig(disabledKey); disabled == "true" { - g = nil - } +func saidYesToStatistics(g *grantRecord, share string) bool { if g != nil && !g.Revoked && !g.statisticsOnly() { return g.Policy == cloudwire.CloudTermsPolicy } - share, _ := kv.LoadConfig(statsShareKey) return share == "on" } +// cloudUnderOlderTerms is Cloud on under terms older than the current ones, +// which did not include statistics: nothing goes until the owner accepts. +func cloudUnderOlderTerms(g *grantRecord) bool { + return g != nil && !g.Revoked && !g.statisticsOnly() && g.Policy != cloudwire.CloudTermsPolicy +} + +// ReconcileStatistics makes the grant follow the one answer once it is off: +// the stats purpose and statsId go, and a statistics-only link ends, as +// turning statistics off does. Core calls it after MigrateStatistics, so an +// earlier fleet switch set to off also stops the hourly upload it never +// covered. +func (l *Link) ReconcileStatistics(by string) error { + g, share := l.statisticsAnswer() + if share != "off" || g == nil || g.Revoked || cloudRequiresStatistics(g) || + (len(g.StatsID) == 0 && !slices.Contains(g.Purposes, protocol.CloudPurposeStats)) { + return nil + } + return l.StopStats(by) +} + // cloudRequiresStatistics reports whether the grant is Cloud under the // current terms, where statistics have no separate off switch. func cloudRequiresStatistics(g *grantRecord) bool { diff --git a/go/internal/cloudlink/choice_test.go b/go/internal/cloudlink/choice_test.go index 805383566..1fae0a3b3 100644 --- a/go/internal/cloudlink/choice_test.go +++ b/go/internal/cloudlink/choice_test.go @@ -123,33 +123,20 @@ func TestMigrationKeepsEveryEarlierNo(t *testing.T) { } } -// With Cloud under the current terms, statistics are part of Cloud whatever -// the local answer; under older terms nothing goes until they are accepted. -func TestCloudDecidesStatisticsWhileItIsOn(t *testing.T) { +// Without a running link no stored grant counts, so the box's own answer +// alone decides. The link's answer, with the grant it honours, is tested in +// link_test.go. +func TestWithoutALinkOnlyTheLocalAnswerCounts(t *testing.T) { kv := &memKV{m: map[string]string{statsShareKey: "off"}} cloud := grantRecord{Policy: cloudwire.CloudTermsPolicy, Purposes: []string{protocol.CloudPurposeServe, protocol.CloudPurposeStats}} if err := saveJSON(kv, grantKey, cloud); err != nil { t.Fatal(err) } - if !SharesStatistics(kv) { - t.Fatal("Cloud under the current terms did not share") - } - older := cloud - older.Policy = "ftw-cloud-2026-10-v2" - kv.m[statsShareKey] = "on" - if err := saveJSON(kv, grantKey, older); err != nil { - t.Fatal(err) - } - if SharesStatistics(kv) { - t.Fatal("Cloud under older terms shared") - } - revoked := cloud - revoked.Revoked = true - if err := saveJSON(kv, grantKey, revoked); err != nil { - t.Fatal(err) - } - if !SharesStatistics(kv) { - t.Fatal("a box leaving Cloud lost its own answer") + for share, want := range map[string][2]bool{"off": {false, false}, "default": {true, false}, "on": {true, true}} { + kv.m[statsShareKey] = share + if got := [2]bool{SharesStatistics(kv), SaidYesToStatistics(kv)}; got != want { + t.Fatalf("%s: shares, said yes = %v, want %v", share, got, want) + } } } diff --git a/go/internal/cloudlink/link.go b/go/internal/cloudlink/link.go index 300588a2e..aad8f1c4b 100644 --- a/go/internal/cloudlink/link.go +++ b/go/internal/cloudlink/link.go @@ -784,7 +784,9 @@ func (l *Link) Stats(req StatsRequest) error { l.mu.Lock() defer l.mu.Unlock() switch { - case l.grant == nil: + case l.grant == nil || l.grant.Revoked || l.disabled: + // The box's own answer, which needs no grant: also while a Cloud + // that was turned off waits for its deletion receipt. if req.Policy != cloudwire.StatsPolicy { return ErrNotGranted } @@ -793,8 +795,8 @@ func (l *Link) Stats(req StatsRequest) error { } l.log.Info("audit: statistics on", "by", req.By) return nil - case l.grant.Revoked: - return ErrPurgePending + case cloudUnderOlderTerms(l.grant): + return ErrCloudTermsFirst case len(l.opts.ServiceKeys) == 0: return ErrNoService } @@ -954,6 +956,9 @@ type Status struct { // StatisticsRequired says it is part of FTW Cloud, with no off switch. Statistics bool `json:"statistics"` StatisticsRequired bool `json:"statisticsRequired"` + // StatisticsNeedTerms is Cloud on under older terms: statistics wait + // until the owner accepts the current ones. + StatisticsNeedTerms bool `json:"statisticsNeedTerms"` // State is the readable copy's: off, on or revoking. State string `json:"state"` StatisticsPolicy string `json:"statisticsPolicy"` @@ -1022,12 +1027,13 @@ func (l *Link) Status() Status { } s.Statistics = sharesStatistics(g, share) s.StatisticsRequired = cloudRequiresStatistics(g) + s.StatisticsNeedTerms = cloudUnderOlderTerms(g) s.StatisticsOnly = l.grant.statisticsOnly() s.CloudTermsAccepted = l.grant != nil && !l.grant.Revoked && l.grant.Policy == cloudwire.CloudTermsPolicy s.StatisticsPolicy = cloudwire.StatsPolicy s.StatisticsReady = l.connected && l.serviceStatsPolicy == cloudwire.StatsPolicy if g := l.grant; g != nil { - s.StatisticsActive = !g.Revoked && g.StatsAtMs > 0 && g.StatsPolicy == cloudwire.StatsPolicy && len(g.StatsID) == 16 && slices.Contains(g.Purposes, protocol.CloudPurposeStats) + s.StatisticsActive = s.Statistics && !g.Revoked && g.StatsAtMs > 0 && g.StatsPolicy == cloudwire.StatsPolicy && len(g.StatsID) == 16 && slices.Contains(g.Purposes, protocol.CloudPurposeStats) } s.BudgetReached = s.BytesLast24h >= budgetBytes if !l.lastSend.IsZero() { diff --git a/go/internal/cloudlink/link_test.go b/go/internal/cloudlink/link_test.go index bb78b8b38..d0ed40611 100644 --- a/go/internal/cloudlink/link_test.go +++ b/go/internal/cloudlink/link_test.go @@ -1147,43 +1147,35 @@ func (h *harness) nextRoster(t *testing.T) cloudwire.Roster { return r } -// Statistics are a second consent under a grant already on. The box draws a -// statsId once, keeps it while statistics stay on, tells the service through -// the roster and sends no statistics yet. -func TestStatisticsAreASecondConsentUnderTheGrant(t *testing.T) { +// Under Cloud, statistics come with the current terms. Under older terms they +// wait: asking for them is refused rather than added beside the old terms. +func TestStatisticsComeWithTheCurrentCloudTerms(t *testing.T) { svc := newFakeService(t) h := newHarness(t, svc) - if err := h.link.Stats(StatsRequest{Policy: "stats-2026-10"}); !errors.Is(err, ErrNotGranted) { - t.Fatalf("statistics with FTW Cloud off: %v", err) - } - h.grant(t) - first := h.nextRoster(t) - svc.waitFor(cloudwire.MsgLive, 5*time.Second) if err := h.link.Stats(StatsRequest{}); !errors.Is(err, ErrGrantRequest) { t.Fatalf("statistics without a policy: %v", err) } - - if err := h.link.Stats(StatsRequest{Policy: "stats-2026-10", By: "app:aaaa1111"}); err != nil { - t.Fatal(err) - } - r := h.nextRoster(t) - if strings.Join(r.Grant.Purposes, ",") != "serve,stats" || r.RosterSeq <= first.RosterSeq { - t.Fatalf("roster after statistics on: purposes %v, seq %d after %d", r.Grant.Purposes, r.RosterSeq, first.RosterSeq) + h.grant(t) + first := h.nextRoster(t) + if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy, By: "app:aaaa1111"}); !errors.Is(err, ErrCloudTermsFirst) { + t.Fatalf("statistics under older terms: %v", err) } - if st := h.link.Status(); strings.Join(st.Purposes, ",") != "serve,stats" { - t.Fatalf("status purposes %v", st.Purposes) + if g := h.record(t); len(g.StatsID) != 0 || strings.Join(g.Purposes, ",") != "serve" { + t.Fatalf("older terms gained statistics: %+v", g) } - g := h.record(t) - if len(g.StatsID) != statsIDBytes || g.StatsPolicy != "stats-2026-10" || g.Policy != "2026-10" { - t.Fatalf("grant record %+v", g) + if st := h.link.Status(); !st.StatisticsNeedTerms || st.Statistics || h.link.SharesStatistics() { + t.Fatalf("status under older terms: %+v", st) } - // A new policy needs a new id and consent start. - if err := h.link.Stats(StatsRequest{Policy: "stats-2026-11"}); err != nil { + if err := h.link.Grant(GrantRequest{Scopes: []string{protocol.ScopeLiveRead}, Policy: cloudwire.CloudTermsPolicy, By: "owner"}); err != nil { t.Fatal(err) } - if again := h.record(t); bytes.Equal(again.StatsID, g.StatsID) || again.StatsPolicy != "stats-2026-11" { - t.Fatalf("statsId %x became %x", g.StatsID, again.StatsID) + r := h.nextRoster(t) + if strings.Join(r.Grant.Purposes, ",") != "serve,stats" || r.RosterSeq <= first.RosterSeq || len(r.Grant.StatsID) != statsIDBytes { + t.Fatalf("roster under the current terms: %+v", r.Grant) + } + if st := h.link.Status(); !st.Statistics || !st.StatisticsRequired || st.StatisticsNeedTerms || !h.link.SaidYesToStatistics() { + t.Fatalf("status under the current terms: %+v", st) } // A restart reads it back. @@ -1192,64 +1184,67 @@ func TestStatisticsAreASecondConsentUnderTheGrant(t *testing.T) { if err != nil { t.Fatal(err) } - if st := reopened.Status(); strings.Join(st.Purposes, ",") != "serve,stats" { + if st := reopened.Status(); strings.Join(st.Purposes, ",") != "serve,stats" || !st.Statistics { t.Fatalf("after a restart: %+v", st) } - for _, env := range svc.received() { if env.T == "stats" { - t.Fatal("the box sent statistics") + t.Fatal("the box sent statistics before the service approved them") } } } -// Turning statistics off forgets the statsId; turning them on again draws a -// new one, because the service refuses an id it has deleted. -func TestStoppingStatisticsForgetsTheID(t *testing.T) { +// While a Cloud that was turned off waits for its deletion receipt, the box's +// own answer can still change. It touches nothing in the grant being deleted. +func TestTheLocalAnswerChangesWhileTheCopyIsBeingDeleted(t *testing.T) { svc := newFakeService(t) h := newHarness(t, svc) - if err := h.link.StopStats("lan"); err != nil { - t.Fatalf("statistics off with FTW Cloud off: %v", err) - } h.grant(t) - h.nextRoster(t) - if err := h.link.Stats(StatsRequest{Policy: "p"}); err != nil { - t.Fatal(err) - } - h.nextRoster(t) - old := h.record(t).StatsID - - if err := h.link.StopStats("lan"); err != nil { + if err := h.link.Revoke("test"); err != nil { t.Fatal(err) } - r := h.nextRoster(t) - if strings.Join(r.Grant.Purposes, ",") != "serve" { - t.Fatalf("roster after statistics off: %v", r.Grant.Purposes) - } - if g := h.record(t); len(g.StatsID) != 0 || g.StatsPolicy != "" || strings.Join(g.Purposes, ",") != "serve" { - t.Fatalf("grant record after statistics off: %+v", g) + if err := h.link.Stats(StatsRequest{Policy: "p"}); !errors.Is(err, ErrNotGranted) { + t.Fatalf("an unknown policy: %v", err) } - if err := h.link.StopStats("lan"); err != nil { - t.Fatalf("statistics off twice: %v", err) + if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy}); err != nil { + t.Fatalf("statistics while revoking: %v", err) } - - if err := h.link.Stats(StatsRequest{Policy: "p"}); err != nil { - t.Fatal(err) + if g := h.record(t); !g.Revoked || len(g.StatsID) != 0 || !h.link.SharesStatistics() { + t.Fatalf("grant while revoking: %+v", g) } - if renewed := h.record(t).StatsID; len(renewed) != statsIDBytes || bytes.Equal(renewed, old) { - t.Fatalf("statsId %x after %x", renewed, old) + if err := h.link.StopStats("lan"); err != nil || h.link.SharesStatistics() { + t.Fatalf("off while revoking: %v", err) } } -func TestNoStatisticsWhileTheCopyIsBeingDeleted(t *testing.T) { +// A stored grant counts only when the link honours it. With a cloud.key that +// is not the one the grant was made with, the box's own answer stands, and a +// box that only shares by default has said no yes. +func TestOnlyAGrantTheLinkHonoursCounts(t *testing.T) { svc := newFakeService(t) h := newHarness(t, svc) - h.grant(t) - if err := h.link.Revoke("test"); err != nil { + if err := h.link.Grant(GrantRequest{Scopes: []string{protocol.ScopeLiveRead}, Policy: cloudwire.CloudTermsPolicy, By: "owner"}); err != nil { t.Fatal(err) } - if err := h.link.Stats(StatsRequest{Policy: "p"}); !errors.Is(err, ErrPurgePending) { - t.Fatalf("statistics while revoking: %v", err) + h.nextRoster(t) + if err := h.kv.SaveConfig(statsShareKey, "default"); err != nil { + t.Fatal(err) + } + if !h.link.SaidYesToStatistics() { + t.Fatal("the current terms are not a yes") + } + if err := os.Remove(h.keyPath); err != nil { + t.Fatal(err) + } + if _, err := createKey(h.keyPath); err != nil { + t.Fatal(err) + } + reopened, err := New(Options{Site: h.site, Roster: fakeRoster{}, KV: h.kv, KeyPath: h.keyPath, ServiceKeys: h.link.opts.ServiceKeys}) + if err != nil { + t.Fatal(err) + } + if reopened.Status().State != "off" || reopened.SaidYesToStatistics() || !reopened.SharesStatistics() { + t.Fatalf("a grant made with another key counted: %+v", reopened.Status()) } } diff --git a/go/internal/cloudlink/stats.go b/go/internal/cloudlink/stats.go index 7edd28e12..9bc239bab 100644 --- a/go/internal/cloudlink/stats.go +++ b/go/internal/cloudlink/stats.go @@ -28,6 +28,11 @@ func (w *writer) statistics(snap grantSnapshot, st *cadence, now time.Time) erro !slices.Contains(g.Purposes, protocol.CloudPurposeStats) || !w.link.clockSynced() { return nil } + // The one answer covers this upload too: off, or Cloud under older + // terms, sends no hour, whatever the grant still carries. + if share, _ := w.link.opts.KV.LoadConfig(statsShareKey); !sharesStatistics(g, share) { + return nil + } hour := now.UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli() if hour < g.StatsAtMs || st.statsHour == hour || now.Sub(st.statsCheckedAt) < time.Minute { return nil diff --git a/go/internal/cloudlink/stats_test.go b/go/internal/cloudlink/stats_test.go index 80b784e74..fb20cc5f1 100644 --- a/go/internal/cloudlink/stats_test.go +++ b/go/internal/cloudlink/stats_test.go @@ -3,9 +3,12 @@ package cloudlink import ( "context" "crypto/ecdsa" - "github.com/srcfl/ftw/go/protocol/cloudwire" + "errors" "testing" "time" + + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" ) type statsSite struct{ *fakeSite } @@ -13,25 +16,54 @@ type statsSite struct{ *fakeSite } func (s statsSite) Statistics(_ context.Context, now time.Time) (cloudwire.Stats, bool) { return cloudwire.Stats{HourMS: now.UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli(), Metrics: map[string]int64{"pv_generation_wh": 120}}, true } -func TestStatisticsSendsSignedClosedHourOnlyWithCurrentConsent(t *testing.T) { + +func statsHarness(t *testing.T, approved bool) (*fakeService, *harness) { svc := newFakeService(t) - svc.statsPolicy = cloudwire.StatsPolicy + if approved { + svc.statsPolicy = cloudwire.StatsPolicy + } h := newHarnessWith(t, svc, []*ecdsa.PublicKey{&svc.key.PublicKey}, nil, func(o *Options) { o.Site = statsSite{&fakeSite{}} o.Now = func() time.Time { return time.Now().Add(-2 * time.Hour) } }) + return svc, h +} + +func sentStats(svc *fakeService) bool { + for _, e := range svc.received() { + if e.T == cloudwire.MsgStats { + return true + } + } + return false +} + +// The hourly upload goes only under the current Cloud terms. Asking for it +// under older terms is refused, and a grant from before this rule that still +// carries the stats purpose under older terms sends nothing either. +func TestStatisticsSendsSignedClosedHourOnlyWithCurrentConsent(t *testing.T) { + svc, h := statsHarness(t, true) h.grant(t) h.nextRoster(t) - if err := h.link.Stats(StatsRequest{Policy: "old-policy"}); err != nil { + if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy}); !errors.Is(err, ErrCloudTermsFirst) { + t.Fatalf("statistics under older terms: %v", err) + } + // As an earlier release could leave it: older terms with the stats purpose. + g := h.record(t) + g.Purposes, g.StatsID, g.StatsPolicy, g.StatsAtMs = []string{protocol.CloudPurposeServe, protocol.CloudPurposeStats}, make([]byte, statsIDBytes), cloudwire.StatsPolicy, 1 + h.link.mu.Lock() + err := h.link.storeLocked(g, "test") + h.link.mu.Unlock() + if err != nil { t.Fatal(err) } h.nextRoster(t) - for _, e := range svc.received() { - if e.T == cloudwire.MsgStats { - t.Fatal("old consent sent statistics") - } + time.Sleep(1500 * time.Millisecond) + if sentStats(svc) { + t.Fatal("older terms sent an hour") } - if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy}); err != nil { + + if err := h.link.Grant(GrantRequest{Scopes: []string{protocol.ScopeLiveRead}, Policy: cloudwire.CloudTermsPolicy, By: "owner"}); err != nil { t.Fatal(err) } r := h.nextRoster(t) @@ -46,33 +78,73 @@ func TestStatisticsSendsSignedClosedHourOnlyWithCurrentConsent(t *testing.T) { if !h.link.Status().StatisticsActive { t.Fatal("current consent inactive") } - if err := h.link.StopStats("box"); err != nil { + if err := h.link.StopStats("box"); !errors.Is(err, ErrCloudNeedsStatistics) { + t.Fatalf("Cloud stopped statistics alone: %v", err) + } + if err := h.link.End("box"); err != nil { t.Fatal(err) } - r = h.nextRoster(t) - if len(r.Grant.StatsID) != 0 || h.link.Status().StatisticsActive { - t.Fatal("withdrawn consent remained active") + if h.link.Status().StatisticsActive { + t.Fatal("ended Cloud kept statistics active") } } -func TestStatisticsWaitsForTheServicesApprovedPolicy(t *testing.T) { - svc := newFakeService(t) - h := newHarnessWith(t, svc, []*ecdsa.PublicKey{&svc.key.PublicKey}, nil, func(o *Options) { - o.Site = statsSite{&fakeSite{}} - o.Now = func() time.Time { return time.Now().Add(-2 * time.Hour) } - }) +// A statistics-only link from a local yes follows the one answer: off sends +// no hour, and ReconcileStatistics ends the link as turning statistics off +// does. +func TestALocalNoStopsTheHourlyUpload(t *testing.T) { + svc, h := statsHarness(t, true) h.grant(t) h.nextRoster(t) - if err := h.link.Stats(StatsRequest{Policy: cloudwire.StatsPolicy}); err != nil { + g := h.record(t) + g.Policy, g.Scopes, g.Purposes = cloudwire.StatsPolicy, []string{}, []string{protocol.CloudPurposeStats} + g.StatsID, g.StatsPolicy, g.StatsAtMs = make([]byte, statsIDBytes), cloudwire.StatsPolicy, 1 + if err := h.kv.SaveConfig(statsShareKey, "off"); err != nil { + t.Fatal(err) + } + h.link.mu.Lock() + err := h.link.storeLocked(g, "test") + h.link.mu.Unlock() + if err != nil { + t.Fatal(err) + } + h.nextRoster(t) + time.Sleep(1500 * time.Millisecond) + if sentStats(svc) { + t.Fatal("an answer of off sent an hour") + } + if st := h.link.Status(); st.Statistics || st.StatisticsActive { + t.Fatalf("status says statistics run: %+v", st) + } + + // The same link with the answer on does send, so the answer is the gate. + if err := h.kv.SaveConfig(statsShareKey, "on"); err != nil { + t.Fatal(err) + } + svc.waitFor(cloudwire.MsgStats, 3*time.Second) + + if err := h.kv.SaveConfig(statsShareKey, "off"); err != nil { + t.Fatal(err) + } + if err := h.link.ReconcileStatistics("test"); err != nil { + t.Fatal(err) + } + if rec := h.record(t); !rec.Revoked || h.link.Status().Link != "ending" { + t.Fatalf("the statistics-only link did not end: %+v %+v", rec, h.link.Status()) + } +} + +func TestStatisticsWaitsForTheServicesApprovedPolicy(t *testing.T) { + svc, h := statsHarness(t, false) + if err := h.link.Grant(GrantRequest{Scopes: []string{protocol.ScopeLiveRead}, Policy: cloudwire.CloudTermsPolicy, By: "owner"}); err != nil { t.Fatal(err) } h.nextRoster(t) if h.link.Status().StatisticsReady { t.Fatal("unapproved service declared ready") } - for _, e := range svc.received() { - if e.T == cloudwire.MsgStats { - t.Fatal("statistics sent without service approval") - } + time.Sleep(time.Second) + if sentStats(svc) { + t.Fatal("statistics sent without service approval") } } diff --git a/go/internal/energyreport/day.go b/go/internal/energyreport/day.go index dc47bb5de..29b48c5c0 100644 --- a/go/internal/energyreport/day.go +++ b/go/internal/energyreport/day.go @@ -15,10 +15,11 @@ type Ledger interface { LoadEnergyHistoryContext(context.Context, state.EnergyHistoryQuery) ([]state.EnergyLedgerPoint, bool, error) } -// usable are the ledger qualities that count as a recorded bucket. A gap, -// reset, invalid read or anything this list does not name makes the day -// incomplete. -var usable = map[string]bool{"measured": true, "recovered": true, "integrated": true, "estimated": true} +// usable are the ledger qualities that count as a recorded bucket. A reset +// is one: a charger's session counter starts again at every plug-in, and the +// ledger covers that interval from the power reading. A gap, an invalid read +// or anything this list does not name makes the day incomplete. +var usable = map[string]bool{"measured": true, "recovered": true, "integrated": true, "estimated": true, "reset": true} // sums maps a ledger flow to its place in energywire.Fields. var sums = map[state.EnergyFlow]int{ @@ -37,7 +38,7 @@ const dayMS = int64(24 * time.Hour / time.Millisecond) // // Whole means: a site meter, and every grid meter, solar, battery and charger // asset that was in use that day recorded every bucket of it, with nothing the -// ledger marks as a gap, reset or invalid read. An asset that arrived or went +// ledger marks as a gap or invalid read. An asset that arrived or went // silent during the day makes it incomplete; one that left before the day // began is not part of it. A box that misses part of a day sends nothing for // it rather than a sum that looks like a quiet day. @@ -57,7 +58,8 @@ func Day(ctx context.Context, l Ledger, start time.Time) (energywire.Report, boo default: continue } - if a.LastSeenMS < from { + // Gone before the day began, or first seen after it ended. + if a.LastSeenMS < from || a.FirstSeenMS >= to { continue } if a.FirstSeenMS > from || a.LastSeenMS < to { diff --git a/go/internal/energyreport/day_test.go b/go/internal/energyreport/day_test.go index 103fccf5f..6cd088e80 100644 --- a/go/internal/energyreport/day_test.go +++ b/go/internal/energyreport/day_test.go @@ -173,3 +173,44 @@ func TestAnAssetRetiredBeforeTheDayIsLeftOut(t *testing.T) { t.Fatalf("%+v ok %v err %v", r, ok, err) } } + +// An asset first seen after the day ended is not part of it. +func TestAnAssetAddedAfterTheDayLeavesItWhole(t *testing.T) { + st := wholeDay(t, nil) + w := 400.0 + for _, at := range []time.Time{day.Add(24*time.Hour + 30*time.Minute), day.Add(24*time.Hour + 32*time.Minute)} { + o := state.EnergyObservation{AssetID: "battery:2/battery", DeviceID: "battery:2", AssetKind: state.AssetBattery, Label: "x", + Flow: state.FlowBatteryCharge, AtMs: at.UnixMilli(), PowerW: &w} + if err := st.RecordTickWithOptionalHistory(nil, nil, []state.EnergyObservation{o}); err != nil { + t.Fatal(err) + } + } + if r, ok, err := Day(context.Background(), st, day); err != nil || !ok || r.BatteryCharge != 6 { + t.Fatalf("%+v ok %v err %v", r, ok, err) + } +} + +// A charger's session counter starts again at every plug-in. The ledger marks +// a reset and covers the interval from the power reading, so the day stays +// whole and the charge is counted. +func TestASessionCounterThatStartsAgainKeepsTheDay(t *testing.T) { + st := wholeDay(t, func(id string, _ time.Time) bool { return id == "charger:1/vehicle_charger" }) + var obs []state.EnergyObservation + first := day.Add(-10 * time.Minute) + for at := first; !at.After(day.Add(24*time.Hour + 10*time.Minute)); at = at.Add(tick) { + session := first + if !at.Before(day.Add(12 * time.Hour)) { + session = day.Add(12 * time.Hour) + } + w, c := 1000.0, 1000*at.Sub(session).Hours() + obs = append(obs, state.EnergyObservation{AssetID: "charger:2/vehicle_charger", DeviceID: "charger:2", AssetKind: state.AssetVehicleCharger, + Label: "x", Flow: state.FlowVehicleCharge, AtMs: at.UnixMilli(), PowerW: &w, CounterWh: &c}) + } + if err := st.RecordTickWithOptionalHistory(nil, nil, obs); err != nil { + t.Fatal(err) + } + r, ok, err := Day(context.Background(), st, day) + if err != nil || !ok || r.VehicleCharge != 24 { + t.Fatalf("%+v ok %v err %v", r, ok, err) + } +} diff --git a/go/internal/statsrelay/statsrelay.go b/go/internal/statsrelay/statsrelay.go index d5f260b74..44c81f515 100644 --- a/go/internal/statsrelay/statsrelay.go +++ b/go/internal/statsrelay/statsrelay.go @@ -20,6 +20,7 @@ import ( "mime" "net/http" "net/url" + "strings" "time" "github.com/srcfl/ftw/go/protocol/ohttp" @@ -76,7 +77,13 @@ func pinned(relay, key string) (string, ohttp.KeyConfig, bool) { return "", ohttp.KeyConfig{}, false } u, err := url.Parse(relay) - if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == reportwire.Authority { + if err != nil { + return "", ohttp.KeyConfig{}, false + } + // Host names compare without case and without a trailing dot, so FTW + // Cloud cannot pass as its own relay under another spelling. + host := strings.TrimSuffix(u.Hostname(), ".") + if u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || strings.EqualFold(host, reportwire.Authority) { return "", ohttp.KeyConfig{}, false } return relay, cfg, true diff --git a/go/internal/statsrelay/statsrelay_test.go b/go/internal/statsrelay/statsrelay_test.go index 8df367a59..00d4559e1 100644 --- a/go/internal/statsrelay/statsrelay_test.go +++ b/go/internal/statsrelay/statsrelay_test.go @@ -53,6 +53,8 @@ func TestAPinIsRefusedUnlessItIsARelayAndAKey(t *testing.T) { "no key": {"https://relay.example.net/ftw", ""}, "plain http": {"http://relay.example.net/ftw", key}, "FTW Cloud itself": {"https://cloud.ftw.energy/v1/ohttp", key}, + "FTW Cloud, cased": {"https://Cloud.FTW.Energy/v1/ohttp", key}, + "FTW Cloud, dot": {"https://cloud.ftw.energy./v1/ohttp", key}, "query": {"https://relay.example.net/ftw?box=1", key}, "credentials": {"https://a:b@relay.example.net/ftw", key}, "bad key": {"https://relay.example.net/ftw", "AAAA"}, diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs index 10db8f931..d2517eb5a 100644 --- a/web/fleet-ping-tab.test.mjs +++ b/web/fleet-ping-tab.test.mjs @@ -196,6 +196,12 @@ describe("the statistics switch", () => { assert.match(line, /no separate off switch/); }); + it("says nothing goes under earlier Cloud terms", () => { + const line = shareLine({ statistics: false, statisticsRequired: false, statisticsNeedTerms: true }); + assert.match(line, /earlier terms/); + assert.match(line, /Nothing is sent until you accept the current terms/); + }); + it("says what on and off mean without Cloud", () => { assert.match(shareLine({ statistics: true, statisticsRequired: false }), /takes effect at once, with no restart/); assert.match(shareLine({ statistics: false, statisticsRequired: false }), /^Off\. Nothing is sent/); diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 1aeade10c..6d3ee1616 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -106,6 +106,9 @@ if (status.statisticsRequired) { return "Part of FTW Cloud, with no separate off switch. To stop, turn FTW Cloud off, then turn statistics off here."; } + if (status.statisticsNeedTerms) { + return "FTW Cloud is on under earlier terms, which did not include statistics. Nothing is sent until you accept the current terms."; + } return status.statistics ? "On. Turning it off takes effect at once, with no restart, and the box keeps the answer." : "Off. Nothing is sent. Turning it on takes effect at once."; @@ -239,7 +242,7 @@ .then(function (v) { sharePolicy = v[1]; box.checked = !!(v[0] && v[0].statistics); - box.disabled = !v[0] || !v[1] || !!v[0].statisticsRequired; + box.disabled = !v[0] || !v[1] || !!v[0].statisticsRequired || !!v[0].statisticsNeedTerms; note.textContent = shareLine(v[0]); }) .catch(function () { note.textContent = shareLine(null); }); From 5a18502c5648dae17fd6fbb7d518b35be77c9723 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 10:09:45 +0200 Subject: [PATCH 6/8] Say what goes beside a sealed report, send hours only on a given yes The settings text and docs now say that a report sent through the relay carries two sealed things beside it: a random number drawn new for each report, and the UTC day it was sent. Settings shows both under each report. The hourly C8 upload now needs a given yes (Cloud under the current terms or a local yes). A box that shares only by default sends no hour. Co-Authored-By: Claude Opus 5.5 --- config.example.yaml | 10 ++++---- docs/architecture.md | 36 ++++++++++++++++++----------- go/internal/cloudlink/stats.go | 8 ++++--- go/internal/cloudlink/stats_test.go | 9 ++++++++ go/internal/fleetping/fleetping.go | 4 +++- web/fleet-ping-tab.test.mjs | 12 +++++++++- web/settings/tabs/fleet.js | 23 ++++++++++++++---- 7 files changed, 76 insertions(+), 26 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 17cb7480d..c8e9d86a7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -226,10 +226,12 @@ app_link: # reports, each alone and at its own time. The fleet report carries six # things: FTW version, release channel, driver types, roughly how much # battery, price zone and roughly how old the install is. The energy report -# carries a closed UTC day's six whole-kWh sums. No id, no key, no serial, no -# site name, no counter. Once a release names a relay, both go through it, so -# FTW Cloud never sees the address; until then the fleet report goes straight -# to `endpoint`, which sees the address, and the energy report is not sent. +# carries a closed UTC day's six whole-kWh sums. No box id, no key, no serial, +# no site name, no counter. Once a release names a relay, both go through it, +# sealed with a random number drawn new for each report and the day it was +# sent, so FTW Cloud never sees the address; until then the fleet report goes +# straight to `endpoint`, which sees the address, and the energy report is not +# sent. # `enabled: false` from an earlier release is read once and kept as "off"; it # is no longer a switch of its own. fleet_ping: diff --git a/docs/architecture.md b/docs/architecture.md index 70bcd14b1..993a3cebd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -614,10 +614,14 @@ The box's anonymous daily report is separate from the Cloud readable-copy grant. Once a day it posts its FTW version and channel, the driver types in use, a battery-capacity bucket, its price zone and an install-age bucket. It answers how many reports arrive and what they describe, which helps size engineering -bets. - -It goes over HTTPS to `https://cloud.ftw.energy/v1/fleet`. It never enters the -Noise session. The HTTP handler validates the fixed payload, adds it to the UTC +bets. It is one of two anonymous daily reports under one statistics answer, with +the energy report beside it (ADR 0008). + +Once a release names an Oblivious HTTP relay (`go/internal/statsrelay`), it goes +through that relay to FTW Cloud's gateway, sealed, so FTW Cloud never sees the +box's address. Until then it goes over HTTPS to +`https://cloud.ftw.energy/v1/fleet`, and only from a box that said yes. It never +enters the Noise session. The HTTP handler validates the fixed payload, adds it to the UTC day's counters and drops it. It keeps 90 calendar days of totals, not raw reports. An operator reads totals only on the loopback metrics listener. An explicitly saved former default resolves to Cloud; a private collector keeps @@ -627,7 +631,11 @@ Fleet totals must not form a household record. The message carries no gateway ID name, no counter and no timestamp — nothing in it says which box sent it — values are bucketed rather than reported, the version travels only when it is a release tag so a developer's build reports as unknown, and the send time is -drawn fresh each day rather than sitting in one slot. +drawn fresh each day rather than sitting in one slot. Through the relay the +sealed request also carries a random number, drawn new for each report so FTW +Cloud counts a report once if it arrives twice, and the UTC day it was sent, so +an old request is refused. Neither names the box, and FTW Cloud keeps a hash of +the number for three days only. Driver names get their own rule, because a driver file's name is whatever the thing that installed it called it, and a household can install its own. A name @@ -674,14 +682,16 @@ Cloud service's loopback stats response. The fields still describe a household, payload remains a quasi-identifier: a beta box in a small price zone with a big battery may be the only one like it, and coarse buckets with a small field set are what keep that population large rather than a proof that it is. The endpoint -also sees the source IP while the request is open, though the Cloud service writes no -request log or source address. And with no id there is no way to dedupe, so the totals count -reports, not unique boxes. - -A failed send is forgotten, never retried. Settings → FTW app → Fleet statistics renders the -exact payload from the same call the sender uses, so the claim is checkable -rather than promised. It is on by default; saving `enabled: false` opts out -without a restart. See [`go/internal/fleetping`](../go/internal/fleetping). +also sees the source IP while the request is open when the report goes straight +to it, though the Cloud service writes no request log or source address; through +the relay it does not. And nothing ties two reports to one box, so the totals +count reports, not unique boxes. + +A failed send is forgotten, never retried. Settings → FTW app → Anonymous +statistics renders the exact payload from the same call the sender uses, and +what goes beside it, so the claim is checkable rather than promised. It follows +the one statistics answer, whose switch is on that screen. See +[`go/internal/fleetping`](../go/internal/fleetping). ## Releases diff --git a/go/internal/cloudlink/stats.go b/go/internal/cloudlink/stats.go index 9bc239bab..88aabd9c2 100644 --- a/go/internal/cloudlink/stats.go +++ b/go/internal/cloudlink/stats.go @@ -28,9 +28,11 @@ func (w *writer) statistics(snap grantSnapshot, st *cadence, now time.Time) erro !slices.Contains(g.Purposes, protocol.CloudPurposeStats) || !w.link.clockSynced() { return nil } - // The one answer covers this upload too: off, or Cloud under older - // terms, sends no hour, whatever the grant still carries. - if share, _ := w.link.opts.KV.LoadConfig(statsShareKey); !sharesStatistics(g, share) { + // The one answer covers this upload too, and only a given yes sends an + // hour: Cloud under the current terms or a local yes. Off, a box that + // shares only by default, or Cloud under older terms sends none, + // whatever the grant still carries. + if share, _ := w.link.opts.KV.LoadConfig(statsShareKey); !saidYesToStatistics(g, share) { return nil } hour := now.UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli() diff --git a/go/internal/cloudlink/stats_test.go b/go/internal/cloudlink/stats_test.go index fb20cc5f1..9bf5ddffb 100644 --- a/go/internal/cloudlink/stats_test.go +++ b/go/internal/cloudlink/stats_test.go @@ -117,6 +117,15 @@ func TestALocalNoStopsTheHourlyUpload(t *testing.T) { t.Fatalf("status says statistics run: %+v", st) } + // Sharing only by default is no given yes, so no hour either. + if err := h.kv.SaveConfig(statsShareKey, "default"); err != nil { + t.Fatal(err) + } + time.Sleep(1500 * time.Millisecond) + if sentStats(svc) { + t.Fatal("an answer that was never given sent an hour") + } + // The same link with the answer on does send, so the answer is the gate. if err := h.kv.SaveConfig(statsShareKey, "on"); err != nil { t.Fatal(err) diff --git a/go/internal/fleetping/fleetping.go b/go/internal/fleetping/fleetping.go index 1a360ee5c..c5e21292f 100644 --- a/go/internal/fleetping/fleetping.go +++ b/go/internal/fleetping/fleetping.go @@ -25,7 +25,9 @@ // that names a relay (package statsrelay) sends the report to FTW Cloud // through it instead, sealed with Oblivious HTTP, so FTW Cloud reads the // report without the address and the relay sees the address without the -// report. Without one, this design makes the *payload* useless as an +// report. The sealed request also carries a random number drawn new for each +// report, and the day it was sent (package reportwire); neither joins two +// pings. Without a relay, this design makes the *payload* useless as an // identifier; it does not make the connection anonymous. The drifting send time is what stops arrival time // becoming a second selector alongside it — though the drift is a random walk // in six-hour steps, so it takes about four days to spread across the clock, diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs index d2517eb5a..c3d9ed151 100644 --- a/web/fleet-ping-tab.test.mjs +++ b/web/fleet-ping-tab.test.mjs @@ -65,6 +65,16 @@ describe("the fleet statistics panel", () => { assert.equal(config.fleet_ping, undefined, "render() wrote a config section"); }); + it("says what goes beside each message through the relay", () => { + // The sealed request carries a random report id and the send day. A + // screen that says "no id, no timestamp" and leaves those out would be + // the claim this tab exists not to make. + const html = render({}); + assert.match(html, /random number, drawn new for each report so FTW Cloud counts it once/); + assert.match(html, /UTC day it was sent/); + assert.match(html, /with what goes beside it/); + }); + it("names both reports and says they never travel together", () => { const html = render({}); assert.match(html, /two anonymous reports a day/); @@ -337,7 +347,7 @@ describe("the daily energy report", () => { it("names the fields, the missing id and the relay in its paragraph", () => { const html = render({}); assert.match(html, /id="fleet-energy-payload"/); - for (const want of [/solar generation/i, /grid import and export/i, /battery charge/i, /vehicle charge/i, /whole kWh/i, /no id/i, /independent relay/i, /skipped/i]) { + for (const want of [/solar generation/i, /grid import and export/i, /battery charge/i, /vehicle charge/i, /whole kWh/i, /no box id/i, /independent relay/i, /skipped/i]) { assert.match(html, want, `the paragraph does not say ${want}`); } }); diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 6d3ee1616..318e3e586 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -198,6 +198,14 @@ exact.className = "fleet-ping-json"; exact.textContent = JSON.stringify(body.payload, null, 2); slot.appendChild(exact); + if (body.through_relay) sealedRows(slot); + } + + // sealedRows shows what goes beside a message sent through the relay, + // inside the sealed request: drawn when it is sent, so not shown as values. + function sealedRows(slot) { + slot.appendChild(row("Report id", "a random number, new for each report")); + slot.appendChild(row("Send day", "the UTC day it is sent")); } function paintEnergy(energy) { @@ -228,6 +236,7 @@ exact.className = "fleet-ping-json"; exact.textContent = JSON.stringify(energy.payload, null, 2); slot.appendChild(exact); + sealedRows(slot); } var sharePolicy = null; @@ -323,23 +332,29 @@ "roughly how much battery, its price zone, and roughly how old the " + "install is. Nothing else — no id, no key, no serial, no site name, no " + "counter, no timestamp — so nothing in the message says which box sent " + - "it. The message itself cannot prove how many unique boxes sent reports.

" + + "it. The message itself cannot prove how many unique boxes sent reports. " + + "Sent through the relay, two things go with it, sealed: a random number, " + + "drawn new for each report so FTW Cloud counts it once, and the UTC day it " + + "was sent. Neither says which box sent it.

" + '

Two limits, plainly. Those six things still describe ' + "you: a beta box in a small price zone with a big battery may be the " + "only one of its kind. And without a relay the receiving service sees the address during " + "the request, the way any website does. What it keeps depends on the " + - "endpoint shown below. Here is the whole message, as this box would send it right now.

" + + "endpoint shown below. Here is the whole message, as this box would send it right now, " + + "with what goes beside it.

" + '
' + '

Daily energy report

' + '

The other report: the date, and the ' + "day's solar generation, grid import and export, battery charge and " + - "discharge, and vehicle charge, each in whole kWh. It carries no id and " + + "discharge, and vehicle charge, each in whole kWh. It carries no box id and " + "nothing from the fleet report, and leaves at its own time, so the two " + "cannot be matched. They never travel together: device kinds and a price " + "zone can point at one home, and the energy beside them would then be " + "that home's. It goes through an independent relay, so FTW Cloud reads " + "the numbers without this box's address, and the relay sees the address " + - "without the numbers. A day with a gap in the energy record is skipped.

" + + "without the numbers. Sealed with it go a random number, drawn new for " + + "each report so FTW Cloud counts it once, and the UTC day it was sent. " + + "A day with a gap in the energy record is skipped.

" + '
' + "
" ); From d57cc6ee0cfe9fe57ddf8decad7afec3e3021777 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 08:30:56 +0200 Subject: [PATCH 7/8] feat(fleetping): say coarsely how the box ran in the anonymous daily report The fleet report (ftw.fleet/2) adds four buckets: the slowest plan of the last 24 hours, how many of those plans fell back, the share of driver polls that failed, and the newest update of the last 7 days. Core reads them from the plans it keeps (planner_diagnostics, PlannerDay), a new 24-hour count of polls in the telemetry store (PollsLastDay), and the updater's status. Each bucket is at most four words wide and describes the last day or week, not the house. A box that does not update itself says none, so the install method does not become a field. fleetwire accepts ftw.fleet/1 and ftw.fleet/2, and a day counts health only from reports that carry it. Same switch as the rest of the report; Settings names the new fields and shows them in the preview. Co-Authored-By: Claude Opus 5.5 --- .changeset/fleet-report-health.md | 5 ++ docs/architecture.md | 15 +++- go/cmd/ftw/fleet_ping.go | 41 ++++++++- go/cmd/ftw/fleet_ping_test.go | 54 +++++++++++- go/cmd/ftw/main.go | 2 +- go/internal/fleetping/fleetping.go | 107 ++++++++++++++++++++++-- go/internal/fleetping/fleetping_test.go | 52 ++++++++++-- go/internal/state/diagnostics.go | 13 +++ go/internal/state/diagnostics_test.go | 27 ++++++ go/internal/telemetry/polls_test.go | 31 +++++++ go/internal/telemetry/store.go | 42 ++++++++++ go/protocol/fleetwire/fleet.go | 68 ++++++++++++--- go/protocol/fleetwire/fleet_test.go | 56 +++++++++++++ go/protocol/reportwire/report_test.go | 3 +- web/fleet-ping-tab.test.mjs | 24 ++++-- web/settings/tabs/fleet.js | 24 +++++- 16 files changed, 520 insertions(+), 44 deletions(-) create mode 100644 .changeset/fleet-report-health.md create mode 100644 go/internal/telemetry/polls_test.go diff --git a/.changeset/fleet-report-health.md b/.changeset/fleet-report-health.md new file mode 100644 index 000000000..e6cc19aab --- /dev/null +++ b/.changeset/fleet-report-health.md @@ -0,0 +1,5 @@ +--- +"ftw": minor +--- + +The anonymous daily fleet report also says, coarsely, how the box ran: how slow its slowest plan of the last day was, how many plans fell back, how often its drivers failed to read, and whether its newest update of the last week failed. Each is a bucket of a few words, under the same switch as the rest of the report. Settings shows them in the message preview. diff --git a/docs/architecture.md b/docs/architecture.md index 993a3cebd..d9fd0d8f9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -617,6 +617,16 @@ how many reports arrive and what they describe, which helps size engineering bets. It is one of two anonymous daily reports under one statistics answer, with the energy report beside it (ADR 0008). +Since `ftw.fleet/2` it also says, in four buckets, how the box ran: the +slowest plan of the last 24 hours (`none`, `0-1s`, `1-5s`, `5s+`), how many of +those plans fell back (`none`, `0`, `1-3`, `4+`), the share of driver polls +that failed (`none`, `0`, `0-1%`, `1%+`) and the newest update of the last 7 +days (`none`, `ok`, `failed`). Core reads them from the plans it already keeps +(`planner_diagnostics`), a 24-hour count of polls in the telemetry store and +the updater's status. A box that does not update itself says `none`, so how it +was installed does not become a field. The service accepts both schemas and +counts health only from reports that carry it. + Once a release names an Oblivious HTTP relay (`go/internal/statsrelay`), it goes through that relay to FTW Cloud's gateway, sealed, so FTW Cloud never sees the box's address. Until then it goes over HTTPS to @@ -681,7 +691,10 @@ Three things this does not fix, and each is said on the Settings screen or the Cloud service's loopback stats response. The fields still describe a household, so the payload remains a quasi-identifier: a beta box in a small price zone with a big battery may be the only one like it, and coarse buckets with a small field set -are what keep that population large rather than a proof that it is. The endpoint +are what keep that population large rather than a proof that it is. The health +buckets are each at most four words wide, describe the last day or week rather +than the house, and most boxes share the common answer; the service keeps each +field's daily count apart, never the combination. The endpoint also sees the source IP while the request is open when the report goes straight to it, though the Cloud service writes no request log or source address; through the relay it does not. And nothing ties two reports to one box, so the totals diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go index c64badd9d..996173f9a 100644 --- a/go/cmd/ftw/fleet_ping.go +++ b/go/cmd/ftw/fleet_ping.go @@ -14,8 +14,10 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/drivers" "github.com/srcfl/ftw/go/internal/fleetping" + "github.com/srcfl/ftw/go/internal/selfupdate" "github.com/srcfl/ftw/go/internal/state" "github.com/srcfl/ftw/go/internal/statsrelay" + "github.com/srcfl/ftw/go/internal/telemetry" ) // The anonymous fleet ping, wired to the box. @@ -65,6 +67,8 @@ func startFleetPing( cfgMu *sync.RWMutex, st *state.Store, answer statsAnswer, + tel *telemetry.Store, + updates *selfupdate.Checker, version string, managedDriverDir string, ) (*fleetping.Pinger, error) { @@ -86,9 +90,12 @@ func startFleetPing( // catalogue at the same time as the config so tomorrow's report and // the Settings preview both see the install. catalogue := fleetCatalogue(st, managedDriverDir) + health := fleetHealth(st, tel, updates, time.Now()) cfgMu.RLock() defer cfgMu.RUnlock() - return fleetping.FactsFromConfig(cfg, version, catalogue, installedAt) + f := fleetping.FactsFromConfig(cfg, version, catalogue, installedAt) + f.Health = health + return f }, Provider: provider, }) @@ -100,6 +107,38 @@ func startFleetPing( return pinger, nil } +// fleetHealth reads how the box ran lately, for the report's health buckets: +// the planner's last 24 hours from the plans it kept, the drivers' last 24 +// hours of polls, and the newest update of the last 7 days. A source that is +// missing, as the updater is on a box that does not update itself, reads as +// nothing to report. +func fleetHealth(st *state.Store, tel *telemetry.Store, updates *selfupdate.Checker, now time.Time) fleetping.Health { + var h fleetping.Health + if st != nil { + plans, slowestMs, fallbacks, err := st.PlannerDay(now.Add(-24 * time.Hour).UnixMilli()) + if err != nil { + slog.Debug("fleet report: planner day unavailable", "err", err) + } else { + h.Plans, h.SlowestSolve, h.Fallbacks = plans, time.Duration(slowestMs*float64(time.Millisecond)), fallbacks + } + } + if tel != nil { + h.Polls, h.PollErrors = tel.PollsLastDay(now) + } + if updates != nil { + u := updates.Status() + if u.Action == "update" && !u.UpdatedAt.IsZero() && now.Sub(u.UpdatedAt) <= 7*24*time.Hour { + switch u.State { + case "done": + h.Update = "ok" + case "failed": + h.Update = "failed" + } + } + } + return h +} + // fleetProvider sends the report to FTW Cloud through the relay when this // build names one, and only then: FTW Cloud then never sees the box's // address. A build without a relay posts straight to /v1/fleet as before. An diff --git a/go/cmd/ftw/fleet_ping_test.go b/go/cmd/ftw/fleet_ping_test.go index f8c323bf2..f249da493 100644 --- a/go/cmd/ftw/fleet_ping_test.go +++ b/go/cmd/ftw/fleet_ping_test.go @@ -13,13 +13,17 @@ import ( "strings" "sync" "testing" + "time" "github.com/srcfl/ftw/go/internal/api" "github.com/srcfl/ftw/go/internal/components" "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/control" "github.com/srcfl/ftw/go/internal/driverrepo" + "github.com/srcfl/ftw/go/internal/fleetping" + "github.com/srcfl/ftw/go/internal/selfupdate" "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" ) // A config saved from the Settings screen does not pass through applyDefaults: @@ -117,7 +121,7 @@ func TestRunningFleetPingSeesADriverInstalledLater(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } @@ -339,7 +343,7 @@ func TestNoNameTheHouseholdChoseReachesThePostedPing(t *testing.T) { // path unchanged. Payload() is what sendOnce marshals and posts. ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } @@ -469,7 +473,7 @@ func TestARepositoryRootInsideTheBundledTreeIsNotTheShippedCatalogue(t *testing. ctx, cancel := context.WithCancel(context.Background()) cancel() - pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active) + pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active) if err != nil { t.Fatalf("startFleetPing: %v", err) } @@ -499,3 +503,47 @@ func slicesContain(haystack []string, needle string) bool { } return false } + +// The report's health comes from the plans the box kept, the drivers' polls +// and the newest update, and leaves only as buckets. A box without an updater +// reads as no update, like a box that had none. +func TestFleetHealthReadsTheBoxsLastDay(t *testing.T) { + store := openStore(t) + now := time.Now() + for i, js := range []string{`{"solver":{"solve_ms":6100,"fallback":true}}`, `{"solver":{"solve_ms":300}}`} { + if err := store.SaveDiagnostic(now.Add(-time.Duration(i+1)*time.Hour).UnixMilli(), "scheduled", "SE3", 0, 96, js); err != nil { + t.Fatal(err) + } + } + tel := telemetry.NewStore() + for range 99 { + tel.RecordDriverTick("inverter") + } + tel.RecordDriverError("meter", "timeout") + updates := selfupdate.New(selfupdate.Config{StatusPath: filepath.Join(t.TempDir(), "update-status.json")}, nil) + if err := updates.WriteStatus(selfupdate.UpdateStatus{State: "failed", Action: "update", Target: "v9.9.9", UpdatedAt: now.Add(-48 * time.Hour)}); err != nil { + t.Fatal(err) + } + + h := fleetHealth(store, tel, updates, now) + if h.Plans != 2 || h.SlowestSolve != 6100*time.Millisecond || h.Fallbacks != 1 || h.Polls != 100 || h.PollErrors != 1 || h.Update != "failed" { + t.Fatalf("%+v", h) + } + p := fleetping.Build(fleetping.Facts{Health: h}, now) + if p.Solve != "5s+" || p.Fallbacks != "1-3" || p.Errors != "1%+" || p.Update != "failed" { + t.Fatalf("%+v", p) + } + + // A week on, the failed update is no longer news; a restart is not an + // update at all. + if h := fleetHealth(store, tel, updates, now.Add(6*24*time.Hour)); h.Update != "" { + t.Fatalf("a week-old update still reads %q", h.Update) + } + _ = updates.WriteStatus(selfupdate.UpdateStatus{State: "done", Action: "restart", UpdatedAt: now}) + if h := fleetHealth(store, tel, updates, now); h.Update != "" { + t.Fatalf("a restart reads as an update: %q", h.Update) + } + if h := fleetHealth(store, tel, nil, now); h.Update != "" || h.Plans != 2 { + t.Fatalf("without an updater: %+v", h) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index e77754b41..aea4e3c3e 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2495,7 +2495,7 @@ func main() { migrateStatistics(cfg, cfgMu, st, cloudLink) answer := statsAnswer{st: st, link: cloudLink} fleetPinger, fleetPingErr := startFleetPing( - ctx, cfg, cfgMu, st, answer, Version, config.ManagedDriversDirOverride, + ctx, cfg, cfgMu, st, answer, tel, selfUpdater, Version, config.ManagedDriversDirOverride, ) if fleetPingErr != nil { slog.Warn("fleet ping unavailable", "err", fleetPingErr) diff --git a/go/internal/fleetping/fleetping.go b/go/internal/fleetping/fleetping.go index c5e21292f..72902df21 100644 --- a/go/internal/fleetping/fleetping.go +++ b/go/internal/fleetping/fleetping.go @@ -5,6 +5,9 @@ // on which version, with which drivers, so engineering effort lands where the // fleet actually is. It runs against the constraint in docs/architecture.md: // the daily totals must not give the relay a stable household record. +// It also says, coarsely, how the box ran over the last day: how slow its +// slowest plan was, whether plans fell back, how often driver polls failed and +// whether an update failed, so the fleet's trouble is found where it is. // // So one ping is a picture of one box's shape and carries nothing that joins // two pings together: @@ -130,6 +133,29 @@ type Facts struct { // InstalledAt is when this box first ran FTW. InstalledAt time.Time + + // Health is how the box ran lately. + Health Health +} + +// Health is how the box ran lately, before it is coarsened. Every field is a +// sum over the whole box: never a driver's name, a plan's time or an +// update's version. +type Health struct { + // Plans is how many plans the planner kept in the last 24 hours, + // SlowestSolve the longest any of them took, and Fallbacks how many were + // a fallback plan. + Plans int + SlowestSolve time.Duration + Fallbacks int + + // Polls is how many driver polls ran in the last 24 hours, all drivers + // together, and PollErrors how many failed. + Polls, PollErrors uint64 + + // Update is the outcome of the newest update in the last 7 days: "ok", + // "failed", or empty without one. + Update string } // Payload is the whole ping. Every field is a bucket or a label shared with @@ -144,14 +170,81 @@ type Payload = fleetwire.Report func Build(f Facts, now time.Time) Payload { version, channel := releaseVersion(f.Version) return Payload{ - Schema: Schema, - Version: version, - Channel: channel, - Drivers: driverTypes(f.Drivers, f.Catalogue), - Battery: batteryBucket(f.BatteryWh), - Zone: knownZone(f.PriceZone), - Age: ageBucket(f.InstalledAt, now), + Schema: Schema, + Version: version, + Channel: channel, + Drivers: driverTypes(f.Drivers, f.Catalogue), + Battery: batteryBucket(f.BatteryWh), + Zone: knownZone(f.PriceZone), + Age: ageBucket(f.InstalledAt, now), + Solve: solveBucket(f.Health), + Fallbacks: fallbackBucket(f.Health), + Errors: errorBucket(f.Health), + Update: updateBucket(f.Health.Update), + } +} + +// The health buckets answer where engineering time goes: is the planner fast +// enough on the hardware boxes run, does it fall back, do drivers fail, do +// updates. Each is three or four words wide and describes the last day, or +// for updates the last week, so it changes as the box runs instead of +// becoming one more fixed trait of the household. Most boxes share the +// common answer, which is what keeps the rarer ones in a crowd. + +// solveBucket is the slowest plan of the last 24 hours. Five seconds is the +// edge that matters: past it a small box's planner runs out of its budget. +func solveBucket(h Health) string { + switch s := h.SlowestSolve; { + case h.Plans <= 0: + return "none" + case s < time.Second: + return "0-1s" + case s < 5*time.Second: + return "1-5s" + default: + return "5s+" + } +} + +// fallbackBucket is how many of the last 24 hours' plans fell back. +func fallbackBucket(h Health) string { + switch { + case h.Plans <= 0: + return "none" + case h.Fallbacks <= 0: + return "0" + case h.Fallbacks <= 3: + return "1-3" + default: + return "4+" + } +} + +// errorBucket is the share of the last 24 hours' driver polls that failed. +// A share, not a count, so a driver that polls every second and one that +// polls every minute land in the same bucket for the same trouble. +func errorBucket(h Health) string { + switch { + case h.Polls == 0: + return "none" + case h.PollErrors == 0: + return "0" + case h.PollErrors*100 < h.Polls: + return "0-1%" + default: + return "1%+" + } +} + +// updateBucket is the newest update of the last 7 days. A box that does not +// update itself says none, like a box that had no update, so how a box was +// installed does not become one more field. +func updateBucket(outcome string) string { + switch outcome { + case "ok", "failed": + return outcome } + return "none" } // releaseTag matches the two shapes this project publishes: vX.Y.Z and diff --git a/go/internal/fleetping/fleetping_test.go b/go/internal/fleetping/fleetping_test.go index b5ecb5e2c..e62708daa 100644 --- a/go/internal/fleetping/fleetping_test.go +++ b/go/internal/fleetping/fleetping_test.go @@ -17,6 +17,7 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest" + "github.com/srcfl/ftw/go/protocol/fleetwire" ) // The whole point of this package is what it does NOT send. These tests are @@ -53,12 +54,16 @@ func TestPayloadCarriesOnlyTheAgreedFields(t *testing.T) { got := marshal(t, Build(facts(), time.Date(2026, 8, 5, 12, 0, 0, 0, time.UTC))) want := map[string]any{ - "schema": "ftw.fleet/1", - "ftw_version": "v1.4.0-beta.2", - "channel": "beta", - "battery_kwh": "5-15", - "price_zone": "SE3", - "install_age": "6-12m", + "schema": "ftw.fleet/2", + "ftw_version": "v1.4.0-beta.2", + "channel": "beta", + "battery_kwh": "5-15", + "price_zone": "SE3", + "install_age": "6-12m", + "plan_solve": "none", + "plan_fallbacks": "none", + "driver_errors": "none", + "update": "none", } for key, value := range want { if got[key] != value { @@ -453,7 +458,7 @@ func TestHTTPProviderPostsTheJSONAndReadsNothingBack(t *testing.T) { if err := provider.Send(context.Background(), Build(facts(), time.Now())); err != nil { t.Fatalf("Send: %v", err) } - if got := body.Load().(string); !strings.Contains(got, `"schema":"ftw.fleet/1"`) { + if got := body.Load().(string); !strings.Contains(got, `"schema":"ftw.fleet/2"`) { t.Fatalf("posted body = %s", got) } if got := agent.Load().(string); got != "ftw-fleet-ping" { @@ -567,3 +572,36 @@ func TestTheReportGoesThroughTheRelayUnchanged(t *testing.T) { t.Fatal("the relay saw the report, or the pinger does not say where it sends") } } + +// ---- health ---- + +func TestHealthTravelsOnlyAsBuckets(t *testing.T) { + // Each case is what a box held over its last day; what leaves is a word + // from a short list, never the count or the time behind it. + for _, c := range []struct { + h Health + solve, fallbacks, errors, update string + }{ + {Health{}, "none", "none", "none", "none"}, + {Health{Plans: 96, SlowestSolve: 420 * time.Millisecond, Polls: 86_400}, "0-1s", "0", "0", "none"}, + {Health{Plans: 96, SlowestSolve: time.Second, Fallbacks: 3, Polls: 1000, PollErrors: 9, Update: "ok"}, "1-5s", "1-3", "0-1%", "ok"}, + {Health{Plans: 12, SlowestSolve: 5 * time.Second, Fallbacks: 4, Polls: 1000, PollErrors: 10, Update: "failed"}, "5s+", "4+", "1%+", "failed"}, + {Health{Plans: 1, SlowestSolve: 47 * time.Second, Polls: 3, PollErrors: 3, Update: "v1.4.0"}, "5s+", "0", "1%+", "none"}, + } { + f := facts() + f.Health = c.h + p := Build(f, time.Now()) + if p.Solve != c.solve || p.Fallbacks != c.fallbacks || p.Errors != c.errors || p.Update != c.update { + t.Errorf("%+v: got %s %s %s %s", c.h, p.Solve, p.Fallbacks, p.Errors, p.Update) + } + raw, _ := json.Marshal(p) + if _, err := fleetwire.Decode(raw); err != nil { + t.Errorf("the service would refuse %s", raw) + } + for _, leak := range []string{"420", "47", "86400", "v1.4.0\"", "1000"} { + if strings.Contains(string(raw), leak) { + t.Errorf("%s carries %s", raw, leak) + } + } + } +} diff --git a/go/internal/state/diagnostics.go b/go/internal/state/diagnostics.go index 5c4eb3625..5c38ef15a 100644 --- a/go/internal/state/diagnostics.go +++ b/go/internal/state/diagnostics.go @@ -86,6 +86,19 @@ func (s *Store) LoadDiagnosticsInRange(sinceMs, untilMs int64, return out, rows.Err() } +// PlannerDay sums the plans kept since sinceMs: how many, the longest any +// took to solve in milliseconds, and how many were a fallback plan. The +// anonymous fleet report coarsens these into buckets; nothing else of a plan +// leaves this query. +func (s *Store) PlannerDay(sinceMs int64) (plans int, slowestMs float64, fallbacks int, err error) { + err = s.db.QueryRow( + `SELECT COUNT(*), + COALESCE(MAX(json_extract(json, '$.solver.solve_ms')), 0), + COALESCE(SUM(json_extract(json, '$.solver.fallback') = 1), 0) + FROM planner_diagnostics WHERE ts_ms >= ?`, sinceMs).Scan(&plans, &slowestMs, &fallbacks) + return plans, slowestMs, fallbacks, err +} + // LoadDiagnosticAt returns the snapshot that was active at tsMs — the // most recent one with ts_ms <= tsMs. This is the correct semantics for // "what plan was driving the EMS at this moment?": if a replan ran at diff --git a/go/internal/state/diagnostics_test.go b/go/internal/state/diagnostics_test.go index 32cd4790d..a09314019 100644 --- a/go/internal/state/diagnostics_test.go +++ b/go/internal/state/diagnostics_test.go @@ -223,3 +223,30 @@ func itoa(i int) string { } return string(buf[pos:]) } + +// The fleet report's planner facts: plans kept since a time, the slowest +// solve and the fallbacks. Older plans and plans without solver details do +// not move the slowest. +func TestPlannerDaySumsThePlansSinceATime(t *testing.T) { + st := openTestStore(t) + day := time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC).UnixMilli() + for i, js := range []string{ + `{"solver":{"solve_ms":90000,"fallback":true}}`, // before the window + `{"solver":{"solve_ms":850}}`, + `{"solver":{"solve_ms":4200,"status":"fallback","fallback":true}}`, + `{"zone":"SE3"}`, + `{"solver":{"solve_ms":12.5,"fallback":false}}`, + } { + ts := day + int64(i-1)*3_600_000 + if err := st.SaveDiagnostic(ts, "scheduled", "SE3", 0, 96, js); err != nil { + t.Fatal(err) + } + } + plans, slowest, fallbacks, err := st.PlannerDay(day) + if err != nil || plans != 4 || slowest != 4200 || fallbacks != 1 { + t.Fatalf("plans %d, slowest %v, fallbacks %d, err %v", plans, slowest, fallbacks, err) + } + if plans, slowest, fallbacks, err := st.PlannerDay(day + 10*3_600_000); err != nil || plans != 0 || slowest != 0 || fallbacks != 0 { + t.Fatalf("an empty window: %d %v %d %v", plans, slowest, fallbacks, err) + } +} diff --git a/go/internal/telemetry/polls_test.go b/go/internal/telemetry/polls_test.go new file mode 100644 index 000000000..9c75b1aa6 --- /dev/null +++ b/go/internal/telemetry/polls_test.go @@ -0,0 +1,31 @@ +package telemetry + +import ( + "testing" + "time" +) + +// The fleet report's driver facts: polls and failed polls over the last 24 +// clock hours, all drivers together, forgetting the hour a day back. +func TestPollsLastDayCountsADayOfPollsAndErrors(t *testing.T) { + s := NewStore() + now := time.Date(2026, 10, 24, 9, 30, 0, 0, time.UTC) + s.mu.Lock() + for i := range 10 { + s.countPollLocked(now.Add(-25*time.Hour), i < 5) // a day and an hour ago: gone + s.countPollLocked(now.Add(-23*time.Hour), i < 2) + s.countPollLocked(now, false) + } + s.mu.Unlock() + if polls, errors := s.PollsLastDay(now); polls != 20 || errors != 2 { + t.Fatalf("polls %d, errors %d", polls, errors) + } + s.RecordDriverTick("inverter") + s.RecordDriverError("meter", "timeout") + if polls, errors := s.PollsLastDay(time.Now()); polls < 2 || errors < 1 { + t.Fatalf("recorded polls not counted: %d %d", polls, errors) + } + if polls, _ := s.PollsLastDay(now.Add(48 * time.Hour)); polls != 0 { + t.Fatalf("two days on, %d polls remain", polls) + } +} diff --git a/go/internal/telemetry/store.go b/go/internal/telemetry/store.go index 109876a2d..a4da8e6d2 100644 --- a/go/internal/telemetry/store.go +++ b/go/internal/telemetry/store.go @@ -324,6 +324,16 @@ type Store struct { // arrivals hear of each finished poll (see Arrived). arrivalsMu sync.Mutex arrivals map[chan struct{}]struct{} + + // pollHours counts driver polls and their errors, all drivers together, + // one slot per clock hour of the last day. Guarded by mu. + pollHours [24]pollHour +} + +// pollHour is one clock hour's driver polls, all drivers together. +type pollHour struct { + hour int64 // Unix hours + polls, errors uint64 } type metricSnap struct { @@ -762,12 +772,44 @@ func (s *Store) RecordDriverTick(name string) { s.mu.Lock() defer s.mu.Unlock() s.driverHealthLocked(name).RecordTick() + s.countPollLocked(time.Now(), false) } func (s *Store) RecordDriverError(name, err string) { s.mu.Lock() defer s.mu.Unlock() s.driverHealthLocked(name).RecordError(err) + s.countPollLocked(time.Now(), true) +} + +// countPollLocked counts one poll in its clock hour, reusing the slot of the +// same hour a day before. +func (s *Store) countPollLocked(now time.Time, failed bool) { + hour := now.Unix() / 3600 + slot := &s.pollHours[hour%int64(len(s.pollHours))] + if slot.hour != hour { + *slot = pollHour{hour: hour} + } + slot.polls++ + if failed { + slot.errors++ + } +} + +// PollsLastDay is how many driver polls ran in the 24 clock hours up to now, +// all drivers together, and how many of them failed. The anonymous fleet +// report coarsens the share into a bucket; no driver's name is counted. +func (s *Store) PollsLastDay(now time.Time) (polls, errors uint64) { + s.mu.RLock() + defer s.mu.RUnlock() + hour := now.Unix() / 3600 + for _, slot := range s.pollHours { + if slot.hour > hour-int64(len(s.pollHours)) && slot.hour <= hour { + polls += slot.polls + errors += slot.errors + } + } + return polls, errors } // Remove drops all in-memory state for a driver: readings, Kalman diff --git a/go/protocol/fleetwire/fleet.go b/go/protocol/fleetwire/fleet.go index 216be8fce..9c5e3fe82 100644 --- a/go/protocol/fleetwire/fleet.go +++ b/go/protocol/fleetwire/fleet.go @@ -10,7 +10,10 @@ import ( "strings" ) -const Schema = "ftw.fleet/1" +// Schema is the report a box sends: the shape of the box and how it ran +// lately. SchemaV1, the shape alone, is still accepted from older boxes. +const Schema = "ftw.fleet/2" +const SchemaV1 = "ftw.fleet/1" const StatsSchema = "ftw.fleet-stats/1" const Meaning = "reports, not unique boxes" const MaxBodyBytes = 4096 @@ -27,13 +30,30 @@ type Report struct { Battery string `json:"battery_kwh"` Zone string `json:"price_zone"` Age string `json:"install_age"` + // How the box ran lately, in Schema only: the slowest plan of the last + // 24 hours, how many plans fell back, the share of driver polls that + // failed, and the newest update of the last 7 days. + Solve string `json:"plan_solve,omitempty"` + Fallbacks string `json:"plan_fallbacks,omitempty"` + Errors string `json:"driver_errors,omitempty"` + Update string `json:"update,omitempty"` } +// The health buckets, each a few values wide. +var ( + SolveBuckets = []string{"none", "0-1s", "1-5s", "5s+"} + FallbackBuckets = []string{"none", "0", "1-3", "4+"} + ErrorBuckets = []string{"none", "0", "0-1%", "1%+"} + UpdateBuckets = []string{"none", "ok", "failed"} +) + var release = regexp.MustCompile(`^v[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}(-beta\.[0-9]{1,4})?$`) var driver = regexp.MustCompile(`^[a-z0-9_]{1,64}$`) var zone = regexp.MustCompile(`^[A-Z0-9-]{2,16}$`) // Decode refuses extra fields, so a sender cannot add a household record. +// A SchemaV1 report has the seven shape fields; a Schema report has the +// four health fields too. func Decode(data []byte) (Report, error) { bad := errors.New("fleet: invalid report") var r Report @@ -41,12 +61,19 @@ func Decode(data []byte) (Report, error) { return r, bad } var fields map[string]json.RawMessage - if json.Unmarshal(data, &fields) != nil || len(fields) != 7 || bytes.Equal(bytes.TrimSpace(fields["drivers"]), []byte("null")) { + if json.Unmarshal(data, &fields) != nil || bytes.Equal(bytes.TrimSpace(fields["drivers"]), []byte("null")) { return r, bad } d := json.NewDecoder(bytes.NewReader(data)) d.DisallowUnknownFields() - if d.Decode(&r) != nil || r.Schema != Schema || (r.Version != "unknown" && !release.MatchString(r.Version)) { + if d.Decode(&r) != nil || (r.Version != "unknown" && !release.MatchString(r.Version)) { + return r, bad + } + switch { + case r.Schema == SchemaV1 && len(fields) == 7: + case r.Schema == Schema && len(fields) == 11 && slices.Contains(SolveBuckets, r.Solve) && slices.Contains(FallbackBuckets, r.Fallbacks) && + slices.Contains(ErrorBuckets, r.Errors) && slices.Contains(UpdateBuckets, r.Update): + default: return r, bad } channel := "stable" @@ -67,16 +94,22 @@ func Decode(data []byte) (Report, error) { return r, nil } -// Day is a day's totals. No individual report survives aggregation. +// Day is a day's totals. No individual report survives aggregation. The +// health counts take only Schema reports, and are absent on a day without +// one. type Day struct { - Date string `json:"date"` - Reports int `json:"reports"` - Versions map[string]int `json:"ftw_versions"` - Channels map[string]int `json:"channels"` - Drivers map[string]int `json:"drivers"` - Battery map[string]int `json:"battery_kwh"` - Zones map[string]int `json:"price_zones"` - Age map[string]int `json:"install_age"` + Date string `json:"date"` + Reports int `json:"reports"` + Versions map[string]int `json:"ftw_versions"` + Channels map[string]int `json:"channels"` + Drivers map[string]int `json:"drivers"` + Battery map[string]int `json:"battery_kwh"` + Zones map[string]int `json:"price_zones"` + Age map[string]int `json:"install_age"` + Solve map[string]int `json:"plan_solve,omitempty"` + Fallbacks map[string]int `json:"plan_fallbacks,omitempty"` + Errors map[string]int `json:"driver_errors,omitempty"` + Update map[string]int `json:"update,omitempty"` } type View struct { @@ -108,6 +141,17 @@ func (d *Day) Add(r Report) bool { inc(d.Battery, r.Battery) inc(d.Zones, r.Zone) inc(d.Age, r.Age) + if r.Schema == Schema { + for _, c := range []struct { + counts *map[string]int + label string + }{{&d.Solve, r.Solve}, {&d.Fallbacks, r.Fallbacks}, {&d.Errors, r.Errors}, {&d.Update, r.Update}} { + if *c.counts == nil { + *c.counts = map[string]int{} + } + inc(*c.counts, c.label) + } + } return true } diff --git a/go/protocol/fleetwire/fleet_test.go b/go/protocol/fleetwire/fleet_test.go index 568e72aee..5975a496d 100644 --- a/go/protocol/fleetwire/fleet_test.go +++ b/go/protocol/fleetwire/fleet_test.go @@ -45,3 +45,59 @@ func TestCountersHaveFixedLimitsAndNoReportSurvives(t *testing.T) { t.Fatal("identifier persisted") } } + +const validV2 = `{"schema":"ftw.fleet/2","ftw_version":"v0.146.0-beta.1","channel":"beta","drivers":["pixii","pixii_pv"],"battery_kwh":"5-15","price_zone":"SE4","install_age":"1-6m","plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"none"}` + +// A report that says how the box ran carries four health buckets, each from +// a short fixed list; one without them is the older shape, and nothing else +// passes. +func TestHealthBucketsAreAFewFixedWords(t *testing.T) { + r, err := Decode([]byte(validV2)) + if err != nil || r.Solve != "1-5s" || r.Fallbacks != "0" || r.Errors != "0-1%" || r.Update != "none" { + t.Fatalf("%+v %v", r, err) + } + for _, bad := range []string{ + strings.Replace(validV2, `"1-5s"`, `"2.3s"`, 1), + strings.Replace(validV2, `"plan_fallbacks":"0"`, `"plan_fallbacks":"7"`, 1), + strings.Replace(validV2, `"0-1%"`, `"0.4%"`, 1), + strings.Replace(validV2, `"update":"none"`, `"update":"v0.145.0"`, 1), + strings.Replace(validV2, `,"update":"none"`, ``, 1), + strings.Replace(validV2, `"plan_solve":"1-5s"`, `"plan_solve":""`, 1), + strings.Replace(validV2, `"ftw.fleet/2"`, `"ftw.fleet/1"`, 1), + strings.Replace(valid, `"ftw.fleet/1"`, `"ftw.fleet/2"`, 1), + strings.TrimSuffix(validV2, "}") + `,"home":"secret"}`, + } { + if _, err := Decode([]byte(bad)); err == nil { + t.Fatalf("accepted %s", bad) + } + } + for _, list := range [][]string{SolveBuckets, FallbackBuckets, ErrorBuckets, UpdateBuckets} { + if len(list) > 4 { + t.Fatalf("%v is wider than four buckets", list) + } + } +} + +// A day counts health only from reports that carry it, and a day without +// any has no health fields at all. +func TestADayCountsHealthOnlyFromReportsThatCarryIt(t *testing.T) { + v1, _ := Decode([]byte(valid)) + v2, _ := Decode([]byte(validV2)) + d := NewDay("2026-10-24") + d.Add(v1) + data, _ := json.Marshal(d) + if strings.Contains(string(data), "plan_solve") || strings.Contains(string(data), `"update"`) { + t.Fatalf("a day of older reports has health fields: %s", data) + } + d.Add(v2) + d.Add(v2) + if d.Reports != 3 || d.Solve["1-5s"] != 2 || d.Fallbacks["0"] != 2 || d.Errors["0-1%"] != 2 || d.Update["none"] != 2 || d.Battery["5-15"] != 3 { + t.Fatalf("%+v", d) + } + // A day stored before health existed takes it once a report brings it. + var old Day + _ = json.Unmarshal([]byte(`{"date":"2026-10-24","reports":1,"ftw_versions":{},"channels":{},"drivers":{},"battery_kwh":{},"price_zones":{},"install_age":{}}`), &old) + if !old.Add(v2) || old.Update["none"] != 1 { + t.Fatalf("%+v", old) + } +} diff --git a/go/protocol/reportwire/report_test.go b/go/protocol/reportwire/report_test.go index 7afe0ec8e..02c0ed802 100644 --- a/go/protocol/reportwire/report_test.go +++ b/go/protocol/reportwire/report_test.go @@ -20,7 +20,8 @@ func TestEveryReportHasOneLength(t *testing.T) { drivers[i] = strings.Repeat(string(rune('a'+i)), 64) } fleet, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v1234.1234.1234-beta.1234", Channel: "unknown", - Drivers: drivers, Battery: "30+", Zone: strings.Repeat("Z", 16), Age: "unknown"}) + Drivers: drivers, Battery: "30+", Zone: strings.Repeat("Z", 16), Age: "unknown", + Solve: "none", Fallbacks: "none", Errors: "none", Update: "failed"}) energy, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", PVGeneration: 1000, GridImport: 1000, GridExport: 1000, BatteryCharge: 1000, BatteryDischarge: 1000, VehicleCharge: 1000}) small, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23"}) diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs index c3d9ed151..84b458077 100644 --- a/web/fleet-ping-tab.test.mjs +++ b/web/fleet-ping-tab.test.mjs @@ -45,13 +45,17 @@ function render(config) { } const payload = { - schema: "ftw.fleet/1", + schema: "ftw.fleet/2", ftw_version: "v1.4.0-beta.2", channel: "beta", drivers: ["easee_cloud", "sungrow"], battery_kwh: "5-15", price_zone: "SE3", install_age: "6-12m", + plan_solve: "1-5s", + plan_fallbacks: "0", + driver_errors: "0-1%", + update: "failed", }; describe("the fleet statistics panel", () => { @@ -85,26 +89,27 @@ describe("the fleet statistics panel", () => { it("names every field the message carries", () => { // This screen is the whole argument for a feature that is on by default, // and the sentence people actually read is the paragraph, not the table - // under it. A paragraph that names four of six fields is the screen + // under it. A paragraph that names some of the fields is the screen // making its case with the awkward ones left out — the price zone above // all, since it is the only field that says roughly where the house is. const html = render({}); - for (const field of [/version/i, /channel/i, /device|driver/i, /battery/i, /price zone/i, /how old/i]) { + for (const field of [/version/i, /channel/i, /device|driver/i, /battery/i, /price zone/i, /how old/i, + /slowest plan/i, /fell back/i, /failed to read/i, /update/i]) { assert.match(html, field, `the paragraph does not name ${field}`); } }); it("counts the fields it names, and the count is the fixture's", () => { - // "six things" is a claim like any other on this screen, and the only one + // "ten things" is a claim like any other on this screen, and the only one // that can go stale in silence. // // What this holds is the sentence against the fixture above, which is a // copy of the payload written here by hand. It is not a guard on the box: - // a seventh field would leave this passing. That guard is + // an eleventh field would leave this passing. That guard is // TestPayloadCarriesOnlyTheAgreedFields in go/internal/fleetping, which - // counts the marshalled fields and fails on the seventh — and whoever + // counts the marshalled fields and fails on the eleventh — and whoever // answers it has to come back through here. - const words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight"]; + const words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven"]; const fields = Object.keys(payload).length - 1; // schema names the format, not the box assert.ok(words[fields], `no number word for ${fields} fields`); assert.match(render({}), new RegExp(words[fields] + " things")); @@ -271,8 +276,13 @@ describe("payloadRows", () => { assert.equal(byKey.battery_kwh.value, "5 to 15 kWh"); assert.equal(byKey.price_zone.value, "SE3"); assert.equal(byKey.install_age.value, "six to twelve months"); + assert.equal(byKey.plan_solve.value, "one to five seconds"); + assert.equal(byKey.plan_fallbacks.value, "none"); + assert.equal(byKey.driver_errors.value, "under 1 in 100"); + assert.equal(byKey.update.value, "failed"); assert.equal(byKey.battery_kwh.label, "Battery size"); + assert.equal(byKey.plan_solve.label, "Slowest plan, last day"); }); it("drops only the schema name", () => { diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 318e3e586..31c82258d 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -32,6 +32,18 @@ battery_kwh: "Battery size", price_zone: "Price zone", install_age: "Time since install", + plan_solve: "Slowest plan, last day", + plan_fallbacks: "Fallback plans, last day", + driver_errors: "Failed driver reads, last day", + update: "Newest update, last week", + }; + + // The health fields, each a few words wide. + var HEALTH = { + plan_solve: { none: "no plan", "0-1s": "under a second", "1-5s": "one to five seconds", "5s+": "five seconds or more" }, + plan_fallbacks: { none: "no plan", "0": "none", "1-3": "one to three", "4+": "four or more" }, + driver_errors: { none: "no reads", "0": "none", "0-1%": "under 1 in 100", "1%+": "1 in 100 or more" }, + update: { none: "none", ok: "succeeded", failed: "failed" }, }; var BATTERY = { @@ -54,6 +66,7 @@ if (Array.isArray(value)) return value.length ? value.join(", ") : "none"; if (key === "battery_kwh" && BATTERY[value]) return BATTERY[value]; if (key === "install_age" && AGE[value]) return AGE[value]; + if (HEALTH[key] && HEALTH[key][value]) return HEALTH[key][value]; if (value === "unknown") return "not known"; return String(value); } @@ -327,16 +340,19 @@ '' + '

Checking…

' + '

Daily fleet report

' + - '

The fleet report carries six things once a day: its ' + + '

The fleet report carries ten things once a day: its ' + "FTW version, its release channel, the kinds of device it talks to, " + - "roughly how much battery, its price zone, and roughly how old the " + - "install is. Nothing else — no id, no key, no serial, no site name, no " + + "roughly how much battery, its price zone, roughly how old the " + + "install is, and four rough marks of how it ran: how slow its slowest " + + "plan of the last day was, how many plans fell back, how often its " + + "drivers failed to read, and whether its newest update of the last week " + + "failed. Nothing else — no id, no key, no serial, no site name, no " + "counter, no timestamp — so nothing in the message says which box sent " + "it. The message itself cannot prove how many unique boxes sent reports. " + "Sent through the relay, two things go with it, sealed: a random number, " + "drawn new for each report so FTW Cloud counts it once, and the UTC day it " + "was sent. Neither says which box sent it.

" + - '

Two limits, plainly. Those six things still describe ' + + '

Two limits, plainly. Those ten things still describe ' + "you: a beta box in a small price zone with a big battery may be the " + "only one of its kind. And without a relay the receiving service sees the address during " + "the request, the way any website does. What it keeps depends on the " + From 6a046735f0d3a84952d14d7c3bc830e62e9247d8 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:15:07 +0200 Subject: [PATCH 8/8] fix(fleetping): count only driver polls, and only once the count covers six hours Review of #1821: OCPP errors fed the poll count through RecordDriverError while OCPP successes did not, so an OCPP-only site with one faulted connector read 1%+. Only the driver registry's poll loop counts now (telemetry CountPoll), a failed poll and a good one alike; health records are not polls. The count lives in memory, so after an update's restart it covered minutes and a start-up burst of failed polls could read as trouble. PollsLastDay now says how much of the day it covers, and the report says none until that is six hours. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 6 ++-- go/cmd/ftw/fleet_ping.go | 12 ++++--- go/cmd/ftw/fleet_ping_test.go | 12 +++++-- go/internal/drivers/registry.go | 3 ++ go/internal/drivers/registry_restart_test.go | 26 ++++++++++++++ go/internal/fleetping/fleetping.go | 6 ++-- go/internal/telemetry/polls_test.go | 36 +++++++++++++++----- go/internal/telemetry/store.go | 26 ++++++++++---- web/settings/tabs/fleet.js | 2 +- 9 files changed, 102 insertions(+), 27 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d9fd0d8f9..8b7e1f327 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -622,8 +622,10 @@ slowest plan of the last 24 hours (`none`, `0-1s`, `1-5s`, `5s+`), how many of those plans fell back (`none`, `0`, `1-3`, `4+`), the share of driver polls that failed (`none`, `0`, `0-1%`, `1%+`) and the newest update of the last 7 days (`none`, `ok`, `failed`). Core reads them from the plans it already keeps -(`planner_diagnostics`), a 24-hour count of polls in the telemetry store and -the updater's status. A box that does not update itself says `none`, so how it +(`planner_diagnostics`), a 24-hour count of the driver registry's polls in +the telemetry store and the updater's status. The poll count lives in memory, +so until it covers six hours, as after a restart, the share says `none`. +An OCPP charger's messages are not polls and are not counted. A box that does not update itself says `none`, so how it was installed does not become a field. The service accepts both schemas and counts health only from reports that carry it. diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go index 996173f9a..37f89fbfd 100644 --- a/go/cmd/ftw/fleet_ping.go +++ b/go/cmd/ftw/fleet_ping.go @@ -109,9 +109,9 @@ func startFleetPing( // fleetHealth reads how the box ran lately, for the report's health buckets: // the planner's last 24 hours from the plans it kept, the drivers' last 24 -// hours of polls, and the newest update of the last 7 days. A source that is -// missing, as the updater is on a box that does not update itself, reads as -// nothing to report. +// hours of polls once the count covers six of them, and the newest update of +// the last 7 days. A source that is missing or too short, as the updater is +// on a box that does not update itself, reads as nothing to report. func fleetHealth(st *state.Store, tel *telemetry.Store, updates *selfupdate.Checker, now time.Time) fleetping.Health { var h fleetping.Health if st != nil { @@ -122,8 +122,12 @@ func fleetHealth(st *state.Store, tel *telemetry.Store, updates *selfupdate.Chec h.Plans, h.SlowestSolve, h.Fallbacks = plans, time.Duration(slowestMs*float64(time.Millisecond)), fallbacks } } + // Counts that cover under six hours, as after an update's restart, say + // too little: a start-up burst of failed polls would read as trouble. if tel != nil { - h.Polls, h.PollErrors = tel.PollsLastDay(now) + if polls, errors, covered := tel.PollsLastDay(now); covered >= 6*time.Hour { + h.Polls, h.PollErrors = polls, errors + } } if updates != nil { u := updates.Status() diff --git a/go/cmd/ftw/fleet_ping_test.go b/go/cmd/ftw/fleet_ping_test.go index f249da493..cfb02f3a0 100644 --- a/go/cmd/ftw/fleet_ping_test.go +++ b/go/cmd/ftw/fleet_ping_test.go @@ -517,15 +517,21 @@ func TestFleetHealthReadsTheBoxsLastDay(t *testing.T) { } tel := telemetry.NewStore() for range 99 { - tel.RecordDriverTick("inverter") + tel.CountPoll(false) } - tel.RecordDriverError("meter", "timeout") + tel.CountPoll(true) updates := selfupdate.New(selfupdate.Config{StatusPath: filepath.Join(t.TempDir(), "update-status.json")}, nil) if err := updates.WriteStatus(selfupdate.UpdateStatus{State: "failed", Action: "update", Target: "v9.9.9", UpdatedAt: now.Add(-48 * time.Hour)}); err != nil { t.Fatal(err) } - h := fleetHealth(store, tel, updates, now) + // Just after a start the poll count covers too little to say anything. + if h := fleetHealth(store, tel, updates, now); h.Polls != 0 || h.PollErrors != 0 || h.Plans != 2 { + t.Fatalf("a fresh count was reported: %+v", h) + } + // Seven hours on, it covers enough; the polls and plans are still in + // their 24 hours. + h := fleetHealth(store, tel, updates, now.Add(7*time.Hour)) if h.Plans != 2 || h.SlowestSolve != 6100*time.Millisecond || h.Fallbacks != 1 || h.Polls != 100 || h.PollErrors != 1 || h.Update != "failed" { t.Fatalf("%+v", h) } diff --git a/go/internal/drivers/registry.go b/go/internal/drivers/registry.go index 07ea3bca4..0d5b11e92 100644 --- a/go/internal/drivers/registry.go +++ b/go/internal/drivers/registry.go @@ -1110,6 +1110,9 @@ func (r *Registry) runLoop(rd *runningDriver) { pollCtx, finishPoll := rd.beginCommand(ctx) returned, err := rd.driver.Poll(pollCtx) finishPoll() + if r.tel != nil { + r.tel.CountPoll(err != nil) + } if err != nil { slog.Warn("driver poll failed", "name", rd.cfg.Name, "err", err) if r.tel != nil { diff --git a/go/internal/drivers/registry_restart_test.go b/go/internal/drivers/registry_restart_test.go index 5f0463fb1..03baa1373 100644 --- a/go/internal/drivers/registry_restart_test.go +++ b/go/internal/drivers/registry_restart_test.go @@ -1332,3 +1332,29 @@ function driver_command(action, w, cmd) end t.Errorf("LastSuccess advanced without any host.emit call: %v", h.LastSuccess) } } + +// The fleet report's share of failed polls counts the poll loop's polls, +// failed or not, and nothing else. +func TestRunLoopCountsPollsAndFailedPolls(t *testing.T) { + r := NewRegistry(telemetry.NewStore()) + src := ` +local n = 0 +function driver_init(config) host.set_poll_interval(50) end +function driver_poll() + n = n + 1 + if n % 2 == 0 then error("meter did not answer") end + return 50 +end +function driver_command(action, w, cmd) end +` + path := writeTestDriver(t, src) + if err := r.Add(context.Background(), config.Driver{Name: "d1", Lua: path}); err != nil { + t.Fatal(err) + } + defer r.Remove("d1") + time.Sleep(400 * time.Millisecond) + polls, errors, _ := r.tel.PollsLastDay(time.Now()) + if polls < 4 || errors < 2 || errors > polls/2+1 { + t.Fatalf("polls %d, failed %d after 400ms of 50ms polls, every second one failing", polls, errors) + } +} diff --git a/go/internal/fleetping/fleetping.go b/go/internal/fleetping/fleetping.go index 72902df21..e3f8db017 100644 --- a/go/internal/fleetping/fleetping.go +++ b/go/internal/fleetping/fleetping.go @@ -150,7 +150,8 @@ type Health struct { Fallbacks int // Polls is how many driver polls ran in the last 24 hours, all drivers - // together, and PollErrors how many failed. + // together, and PollErrors how many failed. Both stay zero while the + // count covers too little of the day to say anything. Polls, PollErrors uint64 // Update is the outcome of the newest update in the last 7 days: "ok", @@ -222,7 +223,8 @@ func fallbackBucket(h Health) string { // errorBucket is the share of the last 24 hours' driver polls that failed. // A share, not a count, so a driver that polls every second and one that -// polls every minute land in the same bucket for the same trouble. +// polls every minute land in the same bucket for the same trouble. None is +// no polls, or too few hours of them, as just after a restart. func errorBucket(h Health) string { switch { case h.Polls == 0: diff --git a/go/internal/telemetry/polls_test.go b/go/internal/telemetry/polls_test.go index 9c75b1aa6..14dca625a 100644 --- a/go/internal/telemetry/polls_test.go +++ b/go/internal/telemetry/polls_test.go @@ -6,10 +6,12 @@ import ( ) // The fleet report's driver facts: polls and failed polls over the last 24 -// clock hours, all drivers together, forgetting the hour a day back. +// clock hours, all drivers together, forgetting the hour a day back, and how +// much of the day the count covers. func TestPollsLastDayCountsADayOfPollsAndErrors(t *testing.T) { s := NewStore() now := time.Date(2026, 10, 24, 9, 30, 0, 0, time.UTC) + s.born = now.Add(-30 * time.Hour) s.mu.Lock() for i := range 10 { s.countPollLocked(now.Add(-25*time.Hour), i < 5) // a day and an hour ago: gone @@ -17,15 +19,33 @@ func TestPollsLastDayCountsADayOfPollsAndErrors(t *testing.T) { s.countPollLocked(now, false) } s.mu.Unlock() - if polls, errors := s.PollsLastDay(now); polls != 20 || errors != 2 { - t.Fatalf("polls %d, errors %d", polls, errors) + if polls, errors, covered := s.PollsLastDay(now); polls != 20 || errors != 2 || covered != 24*time.Hour { + t.Fatalf("polls %d, errors %d, covered %s", polls, errors, covered) + } + if polls, _, _ := s.PollsLastDay(now.Add(48 * time.Hour)); polls != 0 { + t.Fatalf("two days on, %d polls remain", polls) } + s.born = now.Add(-2 * time.Hour) + if _, _, covered := s.PollsLastDay(now); covered != 2*time.Hour { + t.Fatalf("two hours after a restart the count covers %s", covered) + } +} + +// Only the poll loop counts. An OCPP charger's messages record success and +// errors on its health, but they are not polls, so a site whose one driver +// is OCPP has nothing to report rather than a 100 % failure from one +// faulted connector. +func TestOnlyPollsAreCounted(t *testing.T) { + s := NewStore() + s.RecordDriverError("ocpp-charger", "connector faulted") + s.RecordDriverSuccess("ocpp-charger") s.RecordDriverTick("inverter") - s.RecordDriverError("meter", "timeout") - if polls, errors := s.PollsLastDay(time.Now()); polls < 2 || errors < 1 { - t.Fatalf("recorded polls not counted: %d %d", polls, errors) + if polls, errors, _ := s.PollsLastDay(time.Now()); polls != 0 || errors != 0 { + t.Fatalf("health records were counted as polls: %d %d", polls, errors) } - if polls, _ := s.PollsLastDay(now.Add(48 * time.Hour)); polls != 0 { - t.Fatalf("two days on, %d polls remain", polls) + s.CountPoll(false) + s.CountPoll(true) + if polls, errors, _ := s.PollsLastDay(time.Now()); polls != 2 || errors != 1 { + t.Fatalf("polls %d, errors %d", polls, errors) } } diff --git a/go/internal/telemetry/store.go b/go/internal/telemetry/store.go index a4da8e6d2..d9dc84309 100644 --- a/go/internal/telemetry/store.go +++ b/go/internal/telemetry/store.go @@ -326,8 +326,9 @@ type Store struct { arrivals map[chan struct{}]struct{} // pollHours counts driver polls and their errors, all drivers together, - // one slot per clock hour of the last day. Guarded by mu. + // one slot per clock hour of the last day, since born. Guarded by mu. pollHours [24]pollHour + born time.Time } // pollHour is one clock hour's driver polls, all drivers together. @@ -351,6 +352,7 @@ func NewStore() *Store { filters: make(map[string]*KalmanFilter1D), health: make(map[string]*DriverHealth), latestMetric: make(map[string]metricSnap), + born: time.Now(), processNoise: 100, // W of expected change between samples measurementNoise: 50, // W of sensor noise } @@ -772,14 +774,21 @@ func (s *Store) RecordDriverTick(name string) { s.mu.Lock() defer s.mu.Unlock() s.driverHealthLocked(name).RecordTick() - s.countPollLocked(time.Now(), false) } func (s *Store) RecordDriverError(name, err string) { s.mu.Lock() defer s.mu.Unlock() s.driverHealthLocked(name).RecordError(err) - s.countPollLocked(time.Now(), true) +} + +// CountPoll counts one finished driver poll, failed or not. The driver +// registry's poll loop is the one caller: an OCPP charger's messages are not +// polls and are counted neither way. +func (s *Store) CountPoll(failed bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.countPollLocked(time.Now(), failed) } // countPollLocked counts one poll in its clock hour, reusing the slot of the @@ -797,11 +806,14 @@ func (s *Store) countPollLocked(now time.Time, failed bool) { } // PollsLastDay is how many driver polls ran in the 24 clock hours up to now, -// all drivers together, and how many of them failed. The anonymous fleet -// report coarsens the share into a bucket; no driver's name is counted. -func (s *Store) PollsLastDay(now time.Time) (polls, errors uint64) { +// all drivers together, how many of them failed, and how much of the day the +// count covers: the counts live in memory, so after a restart they cover +// only the time since. The anonymous fleet report coarsens the share into a +// bucket; no driver's name is counted. +func (s *Store) PollsLastDay(now time.Time) (polls, errors uint64, covered time.Duration) { s.mu.RLock() defer s.mu.RUnlock() + covered = min(24*time.Hour, max(0, now.Sub(s.born))) hour := now.Unix() / 3600 for _, slot := range s.pollHours { if slot.hour > hour-int64(len(s.pollHours)) && slot.hour <= hour { @@ -809,7 +821,7 @@ func (s *Store) PollsLastDay(now time.Time) (polls, errors uint64) { errors += slot.errors } } - return polls, errors + return polls, errors, covered } // Remove drops all in-memory state for a driver: readings, Kalman diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js index 31c82258d..de0b974e9 100644 --- a/web/settings/tabs/fleet.js +++ b/web/settings/tabs/fleet.js @@ -42,7 +42,7 @@ var HEALTH = { plan_solve: { none: "no plan", "0-1s": "under a second", "1-5s": "one to five seconds", "5s+": "five seconds or more" }, plan_fallbacks: { none: "no plan", "0": "none", "1-3": "one to three", "4+": "four or more" }, - driver_errors: { none: "no reads", "0": "none", "0-1%": "under 1 in 100", "1%+": "1 in 100 or more" }, + driver_errors: { none: "too few reads to say", "0": "none", "0-1%": "under 1 in 100", "1%+": "1 in 100 or more" }, update: { none: "none", ok: "succeeded", failed: "failed" }, };