diff --git a/.changeset/fleet-report-health.md b/.changeset/fleet-report-health.md
new file mode 100644
index 000000000..e6cc19aab
--- /dev/null
+++ b/.changeset/fleet-report-health.md
@@ -0,0 +1,5 @@
+---
+"ftw": minor
+---
+
+The anonymous daily fleet report also says, coarsely, how the box ran: how slow its slowest plan of the last day was, how many plans fell back, how often its drivers failed to read, and whether its newest update of the last week failed. Each is a bucket of a few words, under the same switch as the rest of the report. Settings shows them in the message preview.
diff --git a/docs/architecture.md b/docs/architecture.md
index 993a3cebd..8b7e1f327 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -617,6 +617,18 @@ how many reports arrive and what they describe, which helps size engineering
bets. It is one of two anonymous daily reports under one statistics answer, with
the energy report beside it (ADR 0008).
+Since `ftw.fleet/2` it also says, in four buckets, how the box ran: the
+slowest plan of the last 24 hours (`none`, `0-1s`, `1-5s`, `5s+`), how many of
+those plans fell back (`none`, `0`, `1-3`, `4+`), the share of driver polls
+that failed (`none`, `0`, `0-1%`, `1%+`) and the newest update of the last 7
+days (`none`, `ok`, `failed`). Core reads them from the plans it already keeps
+(`planner_diagnostics`), a 24-hour count of the driver registry's polls in
+the telemetry store and the updater's status. The poll count lives in memory,
+so until it covers six hours, as after a restart, the share says `none`.
+An OCPP charger's messages are not polls and are not counted. A box that does not update itself says `none`, so how it
+was installed does not become a field. The service accepts both schemas and
+counts health only from reports that carry it.
+
Once a release names an Oblivious HTTP relay (`go/internal/statsrelay`), it goes
through that relay to FTW Cloud's gateway, sealed, so FTW Cloud never sees the
box's address. Until then it goes over HTTPS to
@@ -681,7 +693,10 @@ Three things this does not fix, and each is said on the Settings screen or the
Cloud service's loopback stats response. The fields still describe a household, so the
payload remains a quasi-identifier: a beta box in a small price zone with a big
battery may be the only one like it, and coarse buckets with a small field set
-are what keep that population large rather than a proof that it is. The endpoint
+are what keep that population large rather than a proof that it is. The health
+buckets are each at most four words wide, describe the last day or week rather
+than the house, and most boxes share the common answer; the service keeps each
+field's daily count apart, never the combination. The endpoint
also sees the source IP while the request is open when the report goes straight
to it, though the Cloud service writes no request log or source address; through
the relay it does not. And nothing ties two reports to one box, so the totals
diff --git a/go/cmd/ftw/fleet_ping.go b/go/cmd/ftw/fleet_ping.go
index c64badd9d..37f89fbfd 100644
--- a/go/cmd/ftw/fleet_ping.go
+++ b/go/cmd/ftw/fleet_ping.go
@@ -14,8 +14,10 @@ import (
"github.com/srcfl/ftw/go/internal/config"
"github.com/srcfl/ftw/go/internal/drivers"
"github.com/srcfl/ftw/go/internal/fleetping"
+ "github.com/srcfl/ftw/go/internal/selfupdate"
"github.com/srcfl/ftw/go/internal/state"
"github.com/srcfl/ftw/go/internal/statsrelay"
+ "github.com/srcfl/ftw/go/internal/telemetry"
)
// The anonymous fleet ping, wired to the box.
@@ -65,6 +67,8 @@ func startFleetPing(
cfgMu *sync.RWMutex,
st *state.Store,
answer statsAnswer,
+ tel *telemetry.Store,
+ updates *selfupdate.Checker,
version string,
managedDriverDir string,
) (*fleetping.Pinger, error) {
@@ -86,9 +90,12 @@ func startFleetPing(
// catalogue at the same time as the config so tomorrow's report and
// the Settings preview both see the install.
catalogue := fleetCatalogue(st, managedDriverDir)
+ health := fleetHealth(st, tel, updates, time.Now())
cfgMu.RLock()
defer cfgMu.RUnlock()
- return fleetping.FactsFromConfig(cfg, version, catalogue, installedAt)
+ f := fleetping.FactsFromConfig(cfg, version, catalogue, installedAt)
+ f.Health = health
+ return f
},
Provider: provider,
})
@@ -100,6 +107,42 @@ func startFleetPing(
return pinger, nil
}
+// fleetHealth reads how the box ran lately, for the report's health buckets:
+// the planner's last 24 hours from the plans it kept, the drivers' last 24
+// hours of polls once the count covers six of them, and the newest update of
+// the last 7 days. A source that is missing or too short, as the updater is
+// on a box that does not update itself, reads as nothing to report.
+func fleetHealth(st *state.Store, tel *telemetry.Store, updates *selfupdate.Checker, now time.Time) fleetping.Health {
+ var h fleetping.Health
+ if st != nil {
+ plans, slowestMs, fallbacks, err := st.PlannerDay(now.Add(-24 * time.Hour).UnixMilli())
+ if err != nil {
+ slog.Debug("fleet report: planner day unavailable", "err", err)
+ } else {
+ h.Plans, h.SlowestSolve, h.Fallbacks = plans, time.Duration(slowestMs*float64(time.Millisecond)), fallbacks
+ }
+ }
+ // Counts that cover under six hours, as after an update's restart, say
+ // too little: a start-up burst of failed polls would read as trouble.
+ if tel != nil {
+ if polls, errors, covered := tel.PollsLastDay(now); covered >= 6*time.Hour {
+ h.Polls, h.PollErrors = polls, errors
+ }
+ }
+ if updates != nil {
+ u := updates.Status()
+ if u.Action == "update" && !u.UpdatedAt.IsZero() && now.Sub(u.UpdatedAt) <= 7*24*time.Hour {
+ switch u.State {
+ case "done":
+ h.Update = "ok"
+ case "failed":
+ h.Update = "failed"
+ }
+ }
+ }
+ return h
+}
+
// fleetProvider sends the report to FTW Cloud through the relay when this
// build names one, and only then: FTW Cloud then never sees the box's
// address. A build without a relay posts straight to /v1/fleet as before. An
diff --git a/go/cmd/ftw/fleet_ping_test.go b/go/cmd/ftw/fleet_ping_test.go
index f8c323bf2..cfb02f3a0 100644
--- a/go/cmd/ftw/fleet_ping_test.go
+++ b/go/cmd/ftw/fleet_ping_test.go
@@ -13,13 +13,17 @@ import (
"strings"
"sync"
"testing"
+ "time"
"github.com/srcfl/ftw/go/internal/api"
"github.com/srcfl/ftw/go/internal/components"
"github.com/srcfl/ftw/go/internal/config"
"github.com/srcfl/ftw/go/internal/control"
"github.com/srcfl/ftw/go/internal/driverrepo"
+ "github.com/srcfl/ftw/go/internal/fleetping"
+ "github.com/srcfl/ftw/go/internal/selfupdate"
"github.com/srcfl/ftw/go/internal/state"
+ "github.com/srcfl/ftw/go/internal/telemetry"
)
// A config saved from the Settings screen does not pass through applyDefaults:
@@ -117,7 +121,7 @@ func TestRunningFleetPingSeesADriverInstalledLater(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
- pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+ pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
if err != nil {
t.Fatalf("startFleetPing: %v", err)
}
@@ -339,7 +343,7 @@ func TestNoNameTheHouseholdChoseReachesThePostedPing(t *testing.T) {
// path unchanged. Payload() is what sendOnce marshals and posts.
ctx, cancel := context.WithCancel(context.Background())
cancel()
- pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+ pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
if err != nil {
t.Fatalf("startFleetPing: %v", err)
}
@@ -469,7 +473,7 @@ func TestARepositoryRootInsideTheBundledTreeIsNotTheShippedCatalogue(t *testing.
ctx, cancel := context.WithCancel(context.Background())
cancel()
- pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+ pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
if err != nil {
t.Fatalf("startFleetPing: %v", err)
}
@@ -499,3 +503,53 @@ func slicesContain(haystack []string, needle string) bool {
}
return false
}
+
+// The report's health comes from the plans the box kept, the drivers' polls
+// and the newest update, and leaves only as buckets. A box without an updater
+// reads as no update, like a box that had none.
+func TestFleetHealthReadsTheBoxsLastDay(t *testing.T) {
+ store := openStore(t)
+ now := time.Now()
+ for i, js := range []string{`{"solver":{"solve_ms":6100,"fallback":true}}`, `{"solver":{"solve_ms":300}}`} {
+ if err := store.SaveDiagnostic(now.Add(-time.Duration(i+1)*time.Hour).UnixMilli(), "scheduled", "SE3", 0, 96, js); err != nil {
+ t.Fatal(err)
+ }
+ }
+ tel := telemetry.NewStore()
+ for range 99 {
+ tel.CountPoll(false)
+ }
+ tel.CountPoll(true)
+ updates := selfupdate.New(selfupdate.Config{StatusPath: filepath.Join(t.TempDir(), "update-status.json")}, nil)
+ if err := updates.WriteStatus(selfupdate.UpdateStatus{State: "failed", Action: "update", Target: "v9.9.9", UpdatedAt: now.Add(-48 * time.Hour)}); err != nil {
+ t.Fatal(err)
+ }
+
+ // Just after a start the poll count covers too little to say anything.
+ if h := fleetHealth(store, tel, updates, now); h.Polls != 0 || h.PollErrors != 0 || h.Plans != 2 {
+ t.Fatalf("a fresh count was reported: %+v", h)
+ }
+ // Seven hours on, it covers enough; the polls and plans are still in
+ // their 24 hours.
+ h := fleetHealth(store, tel, updates, now.Add(7*time.Hour))
+ if h.Plans != 2 || h.SlowestSolve != 6100*time.Millisecond || h.Fallbacks != 1 || h.Polls != 100 || h.PollErrors != 1 || h.Update != "failed" {
+ t.Fatalf("%+v", h)
+ }
+ p := fleetping.Build(fleetping.Facts{Health: h}, now)
+ if p.Solve != "5s+" || p.Fallbacks != "1-3" || p.Errors != "1%+" || p.Update != "failed" {
+ t.Fatalf("%+v", p)
+ }
+
+ // A week on, the failed update is no longer news; a restart is not an
+ // update at all.
+ if h := fleetHealth(store, tel, updates, now.Add(6*24*time.Hour)); h.Update != "" {
+ t.Fatalf("a week-old update still reads %q", h.Update)
+ }
+ _ = updates.WriteStatus(selfupdate.UpdateStatus{State: "done", Action: "restart", UpdatedAt: now})
+ if h := fleetHealth(store, tel, updates, now); h.Update != "" {
+ t.Fatalf("a restart reads as an update: %q", h.Update)
+ }
+ if h := fleetHealth(store, tel, nil, now); h.Update != "" || h.Plans != 2 {
+ t.Fatalf("without an updater: %+v", h)
+ }
+}
diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go
index e77754b41..aea4e3c3e 100644
--- a/go/cmd/ftw/main.go
+++ b/go/cmd/ftw/main.go
@@ -2495,7 +2495,7 @@ func main() {
migrateStatistics(cfg, cfgMu, st, cloudLink)
answer := statsAnswer{st: st, link: cloudLink}
fleetPinger, fleetPingErr := startFleetPing(
- ctx, cfg, cfgMu, st, answer, Version, config.ManagedDriversDirOverride,
+ ctx, cfg, cfgMu, st, answer, tel, selfUpdater, Version, config.ManagedDriversDirOverride,
)
if fleetPingErr != nil {
slog.Warn("fleet ping unavailable", "err", fleetPingErr)
diff --git a/go/internal/drivers/registry.go b/go/internal/drivers/registry.go
index 07ea3bca4..0d5b11e92 100644
--- a/go/internal/drivers/registry.go
+++ b/go/internal/drivers/registry.go
@@ -1110,6 +1110,9 @@ func (r *Registry) runLoop(rd *runningDriver) {
pollCtx, finishPoll := rd.beginCommand(ctx)
returned, err := rd.driver.Poll(pollCtx)
finishPoll()
+ if r.tel != nil {
+ r.tel.CountPoll(err != nil)
+ }
if err != nil {
slog.Warn("driver poll failed", "name", rd.cfg.Name, "err", err)
if r.tel != nil {
diff --git a/go/internal/drivers/registry_restart_test.go b/go/internal/drivers/registry_restart_test.go
index 5f0463fb1..03baa1373 100644
--- a/go/internal/drivers/registry_restart_test.go
+++ b/go/internal/drivers/registry_restart_test.go
@@ -1332,3 +1332,29 @@ function driver_command(action, w, cmd) end
t.Errorf("LastSuccess advanced without any host.emit call: %v", h.LastSuccess)
}
}
+
+// The fleet report's share of failed polls counts the poll loop's polls,
+// failed or not, and nothing else.
+func TestRunLoopCountsPollsAndFailedPolls(t *testing.T) {
+ r := NewRegistry(telemetry.NewStore())
+ src := `
+local n = 0
+function driver_init(config) host.set_poll_interval(50) end
+function driver_poll()
+ n = n + 1
+ if n % 2 == 0 then error("meter did not answer") end
+ return 50
+end
+function driver_command(action, w, cmd) end
+`
+ path := writeTestDriver(t, src)
+ if err := r.Add(context.Background(), config.Driver{Name: "d1", Lua: path}); err != nil {
+ t.Fatal(err)
+ }
+ defer r.Remove("d1")
+ time.Sleep(400 * time.Millisecond)
+ polls, errors, _ := r.tel.PollsLastDay(time.Now())
+ if polls < 4 || errors < 2 || errors > polls/2+1 {
+ t.Fatalf("polls %d, failed %d after 400ms of 50ms polls, every second one failing", polls, errors)
+ }
+}
diff --git a/go/internal/fleetping/fleetping.go b/go/internal/fleetping/fleetping.go
index c5e21292f..e3f8db017 100644
--- a/go/internal/fleetping/fleetping.go
+++ b/go/internal/fleetping/fleetping.go
@@ -5,6 +5,9 @@
// on which version, with which drivers, so engineering effort lands where the
// fleet actually is. It runs against the constraint in docs/architecture.md:
// the daily totals must not give the relay a stable household record.
+// It also says, coarsely, how the box ran over the last day: how slow its
+// slowest plan was, whether plans fell back, how often driver polls failed and
+// whether an update failed, so the fleet's trouble is found where it is.
//
// So one ping is a picture of one box's shape and carries nothing that joins
// two pings together:
@@ -130,6 +133,30 @@ type Facts struct {
// InstalledAt is when this box first ran FTW.
InstalledAt time.Time
+
+ // Health is how the box ran lately.
+ Health Health
+}
+
+// Health is how the box ran lately, before it is coarsened. Every field is a
+// sum over the whole box: never a driver's name, a plan's time or an
+// update's version.
+type Health struct {
+ // Plans is how many plans the planner kept in the last 24 hours,
+ // SlowestSolve the longest any of them took, and Fallbacks how many were
+ // a fallback plan.
+ Plans int
+ SlowestSolve time.Duration
+ Fallbacks int
+
+ // Polls is how many driver polls ran in the last 24 hours, all drivers
+ // together, and PollErrors how many failed. Both stay zero while the
+ // count covers too little of the day to say anything.
+ Polls, PollErrors uint64
+
+ // Update is the outcome of the newest update in the last 7 days: "ok",
+ // "failed", or empty without one.
+ Update string
}
// Payload is the whole ping. Every field is a bucket or a label shared with
@@ -144,14 +171,82 @@ type Payload = fleetwire.Report
func Build(f Facts, now time.Time) Payload {
version, channel := releaseVersion(f.Version)
return Payload{
- Schema: Schema,
- Version: version,
- Channel: channel,
- Drivers: driverTypes(f.Drivers, f.Catalogue),
- Battery: batteryBucket(f.BatteryWh),
- Zone: knownZone(f.PriceZone),
- Age: ageBucket(f.InstalledAt, now),
+ Schema: Schema,
+ Version: version,
+ Channel: channel,
+ Drivers: driverTypes(f.Drivers, f.Catalogue),
+ Battery: batteryBucket(f.BatteryWh),
+ Zone: knownZone(f.PriceZone),
+ Age: ageBucket(f.InstalledAt, now),
+ Solve: solveBucket(f.Health),
+ Fallbacks: fallbackBucket(f.Health),
+ Errors: errorBucket(f.Health),
+ Update: updateBucket(f.Health.Update),
+ }
+}
+
+// The health buckets answer where engineering time goes: is the planner fast
+// enough on the hardware boxes run, does it fall back, do drivers fail, do
+// updates. Each is three or four words wide and describes the last day, or
+// for updates the last week, so it changes as the box runs instead of
+// becoming one more fixed trait of the household. Most boxes share the
+// common answer, which is what keeps the rarer ones in a crowd.
+
+// solveBucket is the slowest plan of the last 24 hours. Five seconds is the
+// edge that matters: past it a small box's planner runs out of its budget.
+func solveBucket(h Health) string {
+ switch s := h.SlowestSolve; {
+ case h.Plans <= 0:
+ return "none"
+ case s < time.Second:
+ return "0-1s"
+ case s < 5*time.Second:
+ return "1-5s"
+ default:
+ return "5s+"
+ }
+}
+
+// fallbackBucket is how many of the last 24 hours' plans fell back.
+func fallbackBucket(h Health) string {
+ switch {
+ case h.Plans <= 0:
+ return "none"
+ case h.Fallbacks <= 0:
+ return "0"
+ case h.Fallbacks <= 3:
+ return "1-3"
+ default:
+ return "4+"
+ }
+}
+
+// errorBucket is the share of the last 24 hours' driver polls that failed.
+// A share, not a count, so a driver that polls every second and one that
+// polls every minute land in the same bucket for the same trouble. None is
+// no polls, or too few hours of them, as just after a restart.
+func errorBucket(h Health) string {
+ switch {
+ case h.Polls == 0:
+ return "none"
+ case h.PollErrors == 0:
+ return "0"
+ case h.PollErrors*100 < h.Polls:
+ return "0-1%"
+ default:
+ return "1%+"
+ }
+}
+
+// updateBucket is the newest update of the last 7 days. A box that does not
+// update itself says none, like a box that had no update, so how a box was
+// installed does not become one more field.
+func updateBucket(outcome string) string {
+ switch outcome {
+ case "ok", "failed":
+ return outcome
}
+ return "none"
}
// releaseTag matches the two shapes this project publishes: vX.Y.Z and
diff --git a/go/internal/fleetping/fleetping_test.go b/go/internal/fleetping/fleetping_test.go
index b5ecb5e2c..e62708daa 100644
--- a/go/internal/fleetping/fleetping_test.go
+++ b/go/internal/fleetping/fleetping_test.go
@@ -17,6 +17,7 @@ import (
"github.com/srcfl/ftw/go/internal/config"
"github.com/srcfl/ftw/go/internal/statsrelay/statsrelaytest"
+ "github.com/srcfl/ftw/go/protocol/fleetwire"
)
// The whole point of this package is what it does NOT send. These tests are
@@ -53,12 +54,16 @@ func TestPayloadCarriesOnlyTheAgreedFields(t *testing.T) {
got := marshal(t, Build(facts(), time.Date(2026, 8, 5, 12, 0, 0, 0, time.UTC)))
want := map[string]any{
- "schema": "ftw.fleet/1",
- "ftw_version": "v1.4.0-beta.2",
- "channel": "beta",
- "battery_kwh": "5-15",
- "price_zone": "SE3",
- "install_age": "6-12m",
+ "schema": "ftw.fleet/2",
+ "ftw_version": "v1.4.0-beta.2",
+ "channel": "beta",
+ "battery_kwh": "5-15",
+ "price_zone": "SE3",
+ "install_age": "6-12m",
+ "plan_solve": "none",
+ "plan_fallbacks": "none",
+ "driver_errors": "none",
+ "update": "none",
}
for key, value := range want {
if got[key] != value {
@@ -453,7 +458,7 @@ func TestHTTPProviderPostsTheJSONAndReadsNothingBack(t *testing.T) {
if err := provider.Send(context.Background(), Build(facts(), time.Now())); err != nil {
t.Fatalf("Send: %v", err)
}
- if got := body.Load().(string); !strings.Contains(got, `"schema":"ftw.fleet/1"`) {
+ if got := body.Load().(string); !strings.Contains(got, `"schema":"ftw.fleet/2"`) {
t.Fatalf("posted body = %s", got)
}
if got := agent.Load().(string); got != "ftw-fleet-ping" {
@@ -567,3 +572,36 @@ func TestTheReportGoesThroughTheRelayUnchanged(t *testing.T) {
t.Fatal("the relay saw the report, or the pinger does not say where it sends")
}
}
+
+// ---- health ----
+
+func TestHealthTravelsOnlyAsBuckets(t *testing.T) {
+ // Each case is what a box held over its last day; what leaves is a word
+ // from a short list, never the count or the time behind it.
+ for _, c := range []struct {
+ h Health
+ solve, fallbacks, errors, update string
+ }{
+ {Health{}, "none", "none", "none", "none"},
+ {Health{Plans: 96, SlowestSolve: 420 * time.Millisecond, Polls: 86_400}, "0-1s", "0", "0", "none"},
+ {Health{Plans: 96, SlowestSolve: time.Second, Fallbacks: 3, Polls: 1000, PollErrors: 9, Update: "ok"}, "1-5s", "1-3", "0-1%", "ok"},
+ {Health{Plans: 12, SlowestSolve: 5 * time.Second, Fallbacks: 4, Polls: 1000, PollErrors: 10, Update: "failed"}, "5s+", "4+", "1%+", "failed"},
+ {Health{Plans: 1, SlowestSolve: 47 * time.Second, Polls: 3, PollErrors: 3, Update: "v1.4.0"}, "5s+", "0", "1%+", "none"},
+ } {
+ f := facts()
+ f.Health = c.h
+ p := Build(f, time.Now())
+ if p.Solve != c.solve || p.Fallbacks != c.fallbacks || p.Errors != c.errors || p.Update != c.update {
+ t.Errorf("%+v: got %s %s %s %s", c.h, p.Solve, p.Fallbacks, p.Errors, p.Update)
+ }
+ raw, _ := json.Marshal(p)
+ if _, err := fleetwire.Decode(raw); err != nil {
+ t.Errorf("the service would refuse %s", raw)
+ }
+ for _, leak := range []string{"420", "47", "86400", "v1.4.0\"", "1000"} {
+ if strings.Contains(string(raw), leak) {
+ t.Errorf("%s carries %s", raw, leak)
+ }
+ }
+ }
+}
diff --git a/go/internal/state/diagnostics.go b/go/internal/state/diagnostics.go
index 5c4eb3625..5c38ef15a 100644
--- a/go/internal/state/diagnostics.go
+++ b/go/internal/state/diagnostics.go
@@ -86,6 +86,19 @@ func (s *Store) LoadDiagnosticsInRange(sinceMs, untilMs int64,
return out, rows.Err()
}
+// PlannerDay sums the plans kept since sinceMs: how many, the longest any
+// took to solve in milliseconds, and how many were a fallback plan. The
+// anonymous fleet report coarsens these into buckets; nothing else of a plan
+// leaves this query.
+func (s *Store) PlannerDay(sinceMs int64) (plans int, slowestMs float64, fallbacks int, err error) {
+ err = s.db.QueryRow(
+ `SELECT COUNT(*),
+ COALESCE(MAX(json_extract(json, '$.solver.solve_ms')), 0),
+ COALESCE(SUM(json_extract(json, '$.solver.fallback') = 1), 0)
+ FROM planner_diagnostics WHERE ts_ms >= ?`, sinceMs).Scan(&plans, &slowestMs, &fallbacks)
+ return plans, slowestMs, fallbacks, err
+}
+
// LoadDiagnosticAt returns the snapshot that was active at tsMs — the
// most recent one with ts_ms <= tsMs. This is the correct semantics for
// "what plan was driving the EMS at this moment?": if a replan ran at
diff --git a/go/internal/state/diagnostics_test.go b/go/internal/state/diagnostics_test.go
index 32cd4790d..a09314019 100644
--- a/go/internal/state/diagnostics_test.go
+++ b/go/internal/state/diagnostics_test.go
@@ -223,3 +223,30 @@ func itoa(i int) string {
}
return string(buf[pos:])
}
+
+// The fleet report's planner facts: plans kept since a time, the slowest
+// solve and the fallbacks. Older plans and plans without solver details do
+// not move the slowest.
+func TestPlannerDaySumsThePlansSinceATime(t *testing.T) {
+ st := openTestStore(t)
+ day := time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC).UnixMilli()
+ for i, js := range []string{
+ `{"solver":{"solve_ms":90000,"fallback":true}}`, // before the window
+ `{"solver":{"solve_ms":850}}`,
+ `{"solver":{"solve_ms":4200,"status":"fallback","fallback":true}}`,
+ `{"zone":"SE3"}`,
+ `{"solver":{"solve_ms":12.5,"fallback":false}}`,
+ } {
+ ts := day + int64(i-1)*3_600_000
+ if err := st.SaveDiagnostic(ts, "scheduled", "SE3", 0, 96, js); err != nil {
+ t.Fatal(err)
+ }
+ }
+ plans, slowest, fallbacks, err := st.PlannerDay(day)
+ if err != nil || plans != 4 || slowest != 4200 || fallbacks != 1 {
+ t.Fatalf("plans %d, slowest %v, fallbacks %d, err %v", plans, slowest, fallbacks, err)
+ }
+ if plans, slowest, fallbacks, err := st.PlannerDay(day + 10*3_600_000); err != nil || plans != 0 || slowest != 0 || fallbacks != 0 {
+ t.Fatalf("an empty window: %d %v %d %v", plans, slowest, fallbacks, err)
+ }
+}
diff --git a/go/internal/telemetry/polls_test.go b/go/internal/telemetry/polls_test.go
new file mode 100644
index 000000000..14dca625a
--- /dev/null
+++ b/go/internal/telemetry/polls_test.go
@@ -0,0 +1,51 @@
+package telemetry
+
+import (
+ "testing"
+ "time"
+)
+
+// The fleet report's driver facts: polls and failed polls over the last 24
+// clock hours, all drivers together, forgetting the hour a day back, and how
+// much of the day the count covers.
+func TestPollsLastDayCountsADayOfPollsAndErrors(t *testing.T) {
+ s := NewStore()
+ now := time.Date(2026, 10, 24, 9, 30, 0, 0, time.UTC)
+ s.born = now.Add(-30 * time.Hour)
+ s.mu.Lock()
+ for i := range 10 {
+ s.countPollLocked(now.Add(-25*time.Hour), i < 5) // a day and an hour ago: gone
+ s.countPollLocked(now.Add(-23*time.Hour), i < 2)
+ s.countPollLocked(now, false)
+ }
+ s.mu.Unlock()
+ if polls, errors, covered := s.PollsLastDay(now); polls != 20 || errors != 2 || covered != 24*time.Hour {
+ t.Fatalf("polls %d, errors %d, covered %s", polls, errors, covered)
+ }
+ if polls, _, _ := s.PollsLastDay(now.Add(48 * time.Hour)); polls != 0 {
+ t.Fatalf("two days on, %d polls remain", polls)
+ }
+ s.born = now.Add(-2 * time.Hour)
+ if _, _, covered := s.PollsLastDay(now); covered != 2*time.Hour {
+ t.Fatalf("two hours after a restart the count covers %s", covered)
+ }
+}
+
+// Only the poll loop counts. An OCPP charger's messages record success and
+// errors on its health, but they are not polls, so a site whose one driver
+// is OCPP has nothing to report rather than a 100 % failure from one
+// faulted connector.
+func TestOnlyPollsAreCounted(t *testing.T) {
+ s := NewStore()
+ s.RecordDriverError("ocpp-charger", "connector faulted")
+ s.RecordDriverSuccess("ocpp-charger")
+ s.RecordDriverTick("inverter")
+ if polls, errors, _ := s.PollsLastDay(time.Now()); polls != 0 || errors != 0 {
+ t.Fatalf("health records were counted as polls: %d %d", polls, errors)
+ }
+ s.CountPoll(false)
+ s.CountPoll(true)
+ if polls, errors, _ := s.PollsLastDay(time.Now()); polls != 2 || errors != 1 {
+ t.Fatalf("polls %d, errors %d", polls, errors)
+ }
+}
diff --git a/go/internal/telemetry/store.go b/go/internal/telemetry/store.go
index 109876a2d..d9dc84309 100644
--- a/go/internal/telemetry/store.go
+++ b/go/internal/telemetry/store.go
@@ -324,6 +324,17 @@ type Store struct {
// arrivals hear of each finished poll (see Arrived).
arrivalsMu sync.Mutex
arrivals map[chan struct{}]struct{}
+
+ // pollHours counts driver polls and their errors, all drivers together,
+ // one slot per clock hour of the last day, since born. Guarded by mu.
+ pollHours [24]pollHour
+ born time.Time
+}
+
+// pollHour is one clock hour's driver polls, all drivers together.
+type pollHour struct {
+ hour int64 // Unix hours
+ polls, errors uint64
}
type metricSnap struct {
@@ -341,6 +352,7 @@ func NewStore() *Store {
filters: make(map[string]*KalmanFilter1D),
health: make(map[string]*DriverHealth),
latestMetric: make(map[string]metricSnap),
+ born: time.Now(),
processNoise: 100, // W of expected change between samples
measurementNoise: 50, // W of sensor noise
}
@@ -770,6 +782,48 @@ func (s *Store) RecordDriverError(name, err string) {
s.driverHealthLocked(name).RecordError(err)
}
+// CountPoll counts one finished driver poll, failed or not. The driver
+// registry's poll loop is the one caller: an OCPP charger's messages are not
+// polls and are counted neither way.
+func (s *Store) CountPoll(failed bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.countPollLocked(time.Now(), failed)
+}
+
+// countPollLocked counts one poll in its clock hour, reusing the slot of the
+// same hour a day before.
+func (s *Store) countPollLocked(now time.Time, failed bool) {
+ hour := now.Unix() / 3600
+ slot := &s.pollHours[hour%int64(len(s.pollHours))]
+ if slot.hour != hour {
+ *slot = pollHour{hour: hour}
+ }
+ slot.polls++
+ if failed {
+ slot.errors++
+ }
+}
+
+// PollsLastDay is how many driver polls ran in the 24 clock hours up to now,
+// all drivers together, how many of them failed, and how much of the day the
+// count covers: the counts live in memory, so after a restart they cover
+// only the time since. The anonymous fleet report coarsens the share into a
+// bucket; no driver's name is counted.
+func (s *Store) PollsLastDay(now time.Time) (polls, errors uint64, covered time.Duration) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ covered = min(24*time.Hour, max(0, now.Sub(s.born)))
+ hour := now.Unix() / 3600
+ for _, slot := range s.pollHours {
+ if slot.hour > hour-int64(len(s.pollHours)) && slot.hour <= hour {
+ polls += slot.polls
+ errors += slot.errors
+ }
+ }
+ return polls, errors, covered
+}
+
// Remove drops all in-memory state for a driver: readings, Kalman
// filters, the health entry and the latest emit_metric snapshots. Called
// from the driver Registry when a driver is removed from config (or
diff --git a/go/protocol/fleetwire/fleet.go b/go/protocol/fleetwire/fleet.go
index 216be8fce..9c5e3fe82 100644
--- a/go/protocol/fleetwire/fleet.go
+++ b/go/protocol/fleetwire/fleet.go
@@ -10,7 +10,10 @@ import (
"strings"
)
-const Schema = "ftw.fleet/1"
+// Schema is the report a box sends: the shape of the box and how it ran
+// lately. SchemaV1, the shape alone, is still accepted from older boxes.
+const Schema = "ftw.fleet/2"
+const SchemaV1 = "ftw.fleet/1"
const StatsSchema = "ftw.fleet-stats/1"
const Meaning = "reports, not unique boxes"
const MaxBodyBytes = 4096
@@ -27,13 +30,30 @@ type Report struct {
Battery string `json:"battery_kwh"`
Zone string `json:"price_zone"`
Age string `json:"install_age"`
+ // How the box ran lately, in Schema only: the slowest plan of the last
+ // 24 hours, how many plans fell back, the share of driver polls that
+ // failed, and the newest update of the last 7 days.
+ Solve string `json:"plan_solve,omitempty"`
+ Fallbacks string `json:"plan_fallbacks,omitempty"`
+ Errors string `json:"driver_errors,omitempty"`
+ Update string `json:"update,omitempty"`
}
+// The health buckets, each a few values wide.
+var (
+ SolveBuckets = []string{"none", "0-1s", "1-5s", "5s+"}
+ FallbackBuckets = []string{"none", "0", "1-3", "4+"}
+ ErrorBuckets = []string{"none", "0", "0-1%", "1%+"}
+ UpdateBuckets = []string{"none", "ok", "failed"}
+)
+
var release = regexp.MustCompile(`^v[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}(-beta\.[0-9]{1,4})?$`)
var driver = regexp.MustCompile(`^[a-z0-9_]{1,64}$`)
var zone = regexp.MustCompile(`^[A-Z0-9-]{2,16}$`)
// Decode refuses extra fields, so a sender cannot add a household record.
+// A SchemaV1 report has the seven shape fields; a Schema report has the
+// four health fields too.
func Decode(data []byte) (Report, error) {
bad := errors.New("fleet: invalid report")
var r Report
@@ -41,12 +61,19 @@ func Decode(data []byte) (Report, error) {
return r, bad
}
var fields map[string]json.RawMessage
- if json.Unmarshal(data, &fields) != nil || len(fields) != 7 || bytes.Equal(bytes.TrimSpace(fields["drivers"]), []byte("null")) {
+ if json.Unmarshal(data, &fields) != nil || bytes.Equal(bytes.TrimSpace(fields["drivers"]), []byte("null")) {
return r, bad
}
d := json.NewDecoder(bytes.NewReader(data))
d.DisallowUnknownFields()
- if d.Decode(&r) != nil || r.Schema != Schema || (r.Version != "unknown" && !release.MatchString(r.Version)) {
+ if d.Decode(&r) != nil || (r.Version != "unknown" && !release.MatchString(r.Version)) {
+ return r, bad
+ }
+ switch {
+ case r.Schema == SchemaV1 && len(fields) == 7:
+ case r.Schema == Schema && len(fields) == 11 && slices.Contains(SolveBuckets, r.Solve) && slices.Contains(FallbackBuckets, r.Fallbacks) &&
+ slices.Contains(ErrorBuckets, r.Errors) && slices.Contains(UpdateBuckets, r.Update):
+ default:
return r, bad
}
channel := "stable"
@@ -67,16 +94,22 @@ func Decode(data []byte) (Report, error) {
return r, nil
}
-// Day is a day's totals. No individual report survives aggregation.
+// Day is a day's totals. No individual report survives aggregation. The
+// health counts take only Schema reports, and are absent on a day without
+// one.
type Day struct {
- Date string `json:"date"`
- Reports int `json:"reports"`
- Versions map[string]int `json:"ftw_versions"`
- Channels map[string]int `json:"channels"`
- Drivers map[string]int `json:"drivers"`
- Battery map[string]int `json:"battery_kwh"`
- Zones map[string]int `json:"price_zones"`
- Age map[string]int `json:"install_age"`
+ Date string `json:"date"`
+ Reports int `json:"reports"`
+ Versions map[string]int `json:"ftw_versions"`
+ Channels map[string]int `json:"channels"`
+ Drivers map[string]int `json:"drivers"`
+ Battery map[string]int `json:"battery_kwh"`
+ Zones map[string]int `json:"price_zones"`
+ Age map[string]int `json:"install_age"`
+ Solve map[string]int `json:"plan_solve,omitempty"`
+ Fallbacks map[string]int `json:"plan_fallbacks,omitempty"`
+ Errors map[string]int `json:"driver_errors,omitempty"`
+ Update map[string]int `json:"update,omitempty"`
}
type View struct {
@@ -108,6 +141,17 @@ func (d *Day) Add(r Report) bool {
inc(d.Battery, r.Battery)
inc(d.Zones, r.Zone)
inc(d.Age, r.Age)
+ if r.Schema == Schema {
+ for _, c := range []struct {
+ counts *map[string]int
+ label string
+ }{{&d.Solve, r.Solve}, {&d.Fallbacks, r.Fallbacks}, {&d.Errors, r.Errors}, {&d.Update, r.Update}} {
+ if *c.counts == nil {
+ *c.counts = map[string]int{}
+ }
+ inc(*c.counts, c.label)
+ }
+ }
return true
}
diff --git a/go/protocol/fleetwire/fleet_test.go b/go/protocol/fleetwire/fleet_test.go
index 568e72aee..5975a496d 100644
--- a/go/protocol/fleetwire/fleet_test.go
+++ b/go/protocol/fleetwire/fleet_test.go
@@ -45,3 +45,59 @@ func TestCountersHaveFixedLimitsAndNoReportSurvives(t *testing.T) {
t.Fatal("identifier persisted")
}
}
+
+const validV2 = `{"schema":"ftw.fleet/2","ftw_version":"v0.146.0-beta.1","channel":"beta","drivers":["pixii","pixii_pv"],"battery_kwh":"5-15","price_zone":"SE4","install_age":"1-6m","plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"none"}`
+
+// A report that says how the box ran carries four health buckets, each from
+// a short fixed list; one without them is the older shape, and nothing else
+// passes.
+func TestHealthBucketsAreAFewFixedWords(t *testing.T) {
+ r, err := Decode([]byte(validV2))
+ if err != nil || r.Solve != "1-5s" || r.Fallbacks != "0" || r.Errors != "0-1%" || r.Update != "none" {
+ t.Fatalf("%+v %v", r, err)
+ }
+ for _, bad := range []string{
+ strings.Replace(validV2, `"1-5s"`, `"2.3s"`, 1),
+ strings.Replace(validV2, `"plan_fallbacks":"0"`, `"plan_fallbacks":"7"`, 1),
+ strings.Replace(validV2, `"0-1%"`, `"0.4%"`, 1),
+ strings.Replace(validV2, `"update":"none"`, `"update":"v0.145.0"`, 1),
+ strings.Replace(validV2, `,"update":"none"`, ``, 1),
+ strings.Replace(validV2, `"plan_solve":"1-5s"`, `"plan_solve":""`, 1),
+ strings.Replace(validV2, `"ftw.fleet/2"`, `"ftw.fleet/1"`, 1),
+ strings.Replace(valid, `"ftw.fleet/1"`, `"ftw.fleet/2"`, 1),
+ strings.TrimSuffix(validV2, "}") + `,"home":"secret"}`,
+ } {
+ if _, err := Decode([]byte(bad)); err == nil {
+ t.Fatalf("accepted %s", bad)
+ }
+ }
+ for _, list := range [][]string{SolveBuckets, FallbackBuckets, ErrorBuckets, UpdateBuckets} {
+ if len(list) > 4 {
+ t.Fatalf("%v is wider than four buckets", list)
+ }
+ }
+}
+
+// A day counts health only from reports that carry it, and a day without
+// any has no health fields at all.
+func TestADayCountsHealthOnlyFromReportsThatCarryIt(t *testing.T) {
+ v1, _ := Decode([]byte(valid))
+ v2, _ := Decode([]byte(validV2))
+ d := NewDay("2026-10-24")
+ d.Add(v1)
+ data, _ := json.Marshal(d)
+ if strings.Contains(string(data), "plan_solve") || strings.Contains(string(data), `"update"`) {
+ t.Fatalf("a day of older reports has health fields: %s", data)
+ }
+ d.Add(v2)
+ d.Add(v2)
+ if d.Reports != 3 || d.Solve["1-5s"] != 2 || d.Fallbacks["0"] != 2 || d.Errors["0-1%"] != 2 || d.Update["none"] != 2 || d.Battery["5-15"] != 3 {
+ t.Fatalf("%+v", d)
+ }
+ // A day stored before health existed takes it once a report brings it.
+ var old Day
+ _ = json.Unmarshal([]byte(`{"date":"2026-10-24","reports":1,"ftw_versions":{},"channels":{},"drivers":{},"battery_kwh":{},"price_zones":{},"install_age":{}}`), &old)
+ if !old.Add(v2) || old.Update["none"] != 1 {
+ t.Fatalf("%+v", old)
+ }
+}
diff --git a/go/protocol/reportwire/report_test.go b/go/protocol/reportwire/report_test.go
index 7afe0ec8e..02c0ed802 100644
--- a/go/protocol/reportwire/report_test.go
+++ b/go/protocol/reportwire/report_test.go
@@ -20,7 +20,8 @@ func TestEveryReportHasOneLength(t *testing.T) {
drivers[i] = strings.Repeat(string(rune('a'+i)), 64)
}
fleet, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v1234.1234.1234-beta.1234", Channel: "unknown",
- Drivers: drivers, Battery: "30+", Zone: strings.Repeat("Z", 16), Age: "unknown"})
+ Drivers: drivers, Battery: "30+", Zone: strings.Repeat("Z", 16), Age: "unknown",
+ Solve: "none", Fallbacks: "none", Errors: "none", Update: "failed"})
energy, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", PVGeneration: 1000, GridImport: 1000,
GridExport: 1000, BatteryCharge: 1000, BatteryDischarge: 1000, VehicleCharge: 1000})
small, _ := json.Marshal(energywire.Report{Schema: energywire.Schema, Date: "2026-10-23"})
diff --git a/web/fleet-ping-tab.test.mjs b/web/fleet-ping-tab.test.mjs
index c3d9ed151..84b458077 100644
--- a/web/fleet-ping-tab.test.mjs
+++ b/web/fleet-ping-tab.test.mjs
@@ -45,13 +45,17 @@ function render(config) {
}
const payload = {
- schema: "ftw.fleet/1",
+ schema: "ftw.fleet/2",
ftw_version: "v1.4.0-beta.2",
channel: "beta",
drivers: ["easee_cloud", "sungrow"],
battery_kwh: "5-15",
price_zone: "SE3",
install_age: "6-12m",
+ plan_solve: "1-5s",
+ plan_fallbacks: "0",
+ driver_errors: "0-1%",
+ update: "failed",
};
describe("the fleet statistics panel", () => {
@@ -85,26 +89,27 @@ describe("the fleet statistics panel", () => {
it("names every field the message carries", () => {
// This screen is the whole argument for a feature that is on by default,
// and the sentence people actually read is the paragraph, not the table
- // under it. A paragraph that names four of six fields is the screen
+ // under it. A paragraph that names some of the fields is the screen
// making its case with the awkward ones left out — the price zone above
// all, since it is the only field that says roughly where the house is.
const html = render({});
- for (const field of [/version/i, /channel/i, /device|driver/i, /battery/i, /price zone/i, /how old/i]) {
+ for (const field of [/version/i, /channel/i, /device|driver/i, /battery/i, /price zone/i, /how old/i,
+ /slowest plan/i, /fell back/i, /failed to read/i, /update/i]) {
assert.match(html, field, `the paragraph does not name ${field}`);
}
});
it("counts the fields it names, and the count is the fixture's", () => {
- // "six things" is a claim like any other on this screen, and the only one
+ // "ten things" is a claim like any other on this screen, and the only one
// that can go stale in silence.
//
// What this holds is the sentence against the fixture above, which is a
// copy of the payload written here by hand. It is not a guard on the box:
- // a seventh field would leave this passing. That guard is
+ // an eleventh field would leave this passing. That guard is
// TestPayloadCarriesOnlyTheAgreedFields in go/internal/fleetping, which
- // counts the marshalled fields and fails on the seventh — and whoever
+ // counts the marshalled fields and fails on the eleventh — and whoever
// answers it has to come back through here.
- const words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight"];
+ const words = ["no", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven"];
const fields = Object.keys(payload).length - 1; // schema names the format, not the box
assert.ok(words[fields], `no number word for ${fields} fields`);
assert.match(render({}), new RegExp(words[fields] + " things"));
@@ -271,8 +276,13 @@ describe("payloadRows", () => {
assert.equal(byKey.battery_kwh.value, "5 to 15 kWh");
assert.equal(byKey.price_zone.value, "SE3");
assert.equal(byKey.install_age.value, "six to twelve months");
+ assert.equal(byKey.plan_solve.value, "one to five seconds");
+ assert.equal(byKey.plan_fallbacks.value, "none");
+ assert.equal(byKey.driver_errors.value, "under 1 in 100");
+ assert.equal(byKey.update.value, "failed");
assert.equal(byKey.battery_kwh.label, "Battery size");
+ assert.equal(byKey.plan_solve.label, "Slowest plan, last day");
});
it("drops only the schema name", () => {
diff --git a/web/settings/tabs/fleet.js b/web/settings/tabs/fleet.js
index 318e3e586..de0b974e9 100644
--- a/web/settings/tabs/fleet.js
+++ b/web/settings/tabs/fleet.js
@@ -32,6 +32,18 @@
battery_kwh: "Battery size",
price_zone: "Price zone",
install_age: "Time since install",
+ plan_solve: "Slowest plan, last day",
+ plan_fallbacks: "Fallback plans, last day",
+ driver_errors: "Failed driver reads, last day",
+ update: "Newest update, last week",
+ };
+
+ // The health fields, each a few words wide.
+ var HEALTH = {
+ plan_solve: { none: "no plan", "0-1s": "under a second", "1-5s": "one to five seconds", "5s+": "five seconds or more" },
+ plan_fallbacks: { none: "no plan", "0": "none", "1-3": "one to three", "4+": "four or more" },
+ driver_errors: { none: "too few reads to say", "0": "none", "0-1%": "under 1 in 100", "1%+": "1 in 100 or more" },
+ update: { none: "none", ok: "succeeded", failed: "failed" },
};
var BATTERY = {
@@ -54,6 +66,7 @@
if (Array.isArray(value)) return value.length ? value.join(", ") : "none";
if (key === "battery_kwh" && BATTERY[value]) return BATTERY[value];
if (key === "install_age" && AGE[value]) return AGE[value];
+ if (HEALTH[key] && HEALTH[key][value]) return HEALTH[key][value];
if (value === "unknown") return "not known";
return String(value);
}
@@ -327,16 +340,19 @@
'' +
'
Checking…
' +
'
Daily fleet report
' +
- '
The fleet report carries six things once a day: its ' +
+ '
The fleet report carries ten things once a day: its ' +
"FTW version, its release channel, the kinds of device it talks to, " +
- "roughly how much battery, its price zone, and roughly how old the " +
- "install is. Nothing else — no id, no key, no serial, no site name, no " +
+ "roughly how much battery, its price zone, roughly how old the " +
+ "install is, and four rough marks of how it ran: how slow its slowest " +
+ "plan of the last day was, how many plans fell back, how often its " +
+ "drivers failed to read, and whether its newest update of the last week " +
+ "failed. Nothing else — no id, no key, no serial, no site name, no " +
"counter, no timestamp — so nothing in the message says which box sent " +
"it. The message itself cannot prove how many unique boxes sent reports. " +
"Sent through the relay, two things go with it, sealed: a random number, " +
"drawn new for each report so FTW Cloud counts it once, and the UTC day it " +
"was sent. Neither says which box sent it.
" +
- '
Two limits, plainly. Those six things still describe ' +
+ '
Two limits, plainly. Those ten things still describe ' +
"you: a beta box in a small price zone with a big battery may be the " +
"only one of its kind. And without a relay the receiving service sees the address during " +
"the request, the way any website does. What it keeps depends on the " +