feat(fleetping): say coarsely how the box ran in the anonymous daily report

.changeset/fleet-report-health.md

@@ -0,0 +1,5 @@
+---
+"ftw": minor
+---
+
+The anonymous daily fleet report also says, coarsely, how the box ran: how slow its slowest plan of the last day was, how many plans fell back, how often its drivers failed to read, and whether its newest update of the last week failed. Each is a bucket of a few words, under the same switch as the rest of the report. Settings shows them in the message preview.

docs/architecture.md

@@ -617,6 +617,18 @@ how many reports arrive and what they describe, which helps size engineering
 bets. It is one of two anonymous daily reports under one statistics answer, with
 the energy report beside it (ADR 0008).
 
+Since `ftw.fleet/2` it also says, in four buckets, how the box ran: the
+slowest plan of the last 24 hours (`none`, `0-1s`, `1-5s`, `5s+`), how many of
+those plans fell back (`none`, `0`, `1-3`, `4+`), the share of driver polls
+that failed (`none`, `0`, `0-1%`, `1%+`) and the newest update of the last 7
+days (`none`, `ok`, `failed`). Core reads them from the plans it already keeps
+(`planner_diagnostics`), a 24-hour count of the driver registry's polls in
+the telemetry store and the updater's status. The poll count lives in memory,
+so until it covers six hours, as after a restart, the share says `none`.
+An OCPP charger's messages are not polls and are not counted. A box that does not update itself says `none`, so how it
+was installed does not become a field. The service accepts both schemas and
+counts health only from reports that carry it.
+
 Once a release names an Oblivious HTTP relay (`go/internal/statsrelay`), it goes
 through that relay to FTW Cloud's gateway, sealed, so FTW Cloud never sees the
 box's address. Until then it goes over HTTPS to
@@ -681,7 +693,10 @@ Three things this does not fix, and each is said on the Settings screen or the
 Cloud service's loopback stats response. The fields still describe a household, so the
 payload remains a quasi-identifier: a beta box in a small price zone with a big
 battery may be the only one like it, and coarse buckets with a small field set
-are what keep that population large rather than a proof that it is. The endpoint
+are what keep that population large rather than a proof that it is. The health
+buckets are each at most four words wide, describe the last day or week rather
+than the house, and most boxes share the common answer; the service keeps each
+field's daily count apart, never the combination. The endpoint
 also sees the source IP while the request is open when the report goes straight
 to it, though the Cloud service writes no request log or source address; through
 the relay it does not. And nothing ties two reports to one box, so the totals

go/cmd/ftw/fleet_ping.go

@@ -14,8 +14,10 @@ import (
 	"github.com/srcfl/ftw/go/internal/config"
 	"github.com/srcfl/ftw/go/internal/drivers"
 	"github.com/srcfl/ftw/go/internal/fleetping"
+	"github.com/srcfl/ftw/go/internal/selfupdate"
 	"github.com/srcfl/ftw/go/internal/state"
 	"github.com/srcfl/ftw/go/internal/statsrelay"
+	"github.com/srcfl/ftw/go/internal/telemetry"
 )
 
 // The anonymous fleet ping, wired to the box.
@@ -65,6 +67,8 @@ func startFleetPing(
 	cfgMu *sync.RWMutex,
 	st *state.Store,
 	answer statsAnswer,
+	tel *telemetry.Store,
+	updates *selfupdate.Checker,
 	version string,
 	managedDriverDir string,
 ) (*fleetping.Pinger, error) {
@@ -86,9 +90,12 @@ func startFleetPing(
 			// catalogue at the same time as the config so tomorrow's report and
 			// the Settings preview both see the install.
 			catalogue := fleetCatalogue(st, managedDriverDir)
+			health := fleetHealth(st, tel, updates, time.Now())
 			cfgMu.RLock()
 			defer cfgMu.RUnlock()
-			return fleetping.FactsFromConfig(cfg, version, catalogue, installedAt)
+			f := fleetping.FactsFromConfig(cfg, version, catalogue, installedAt)
+			f.Health = health
+			return f
 		},
 		Provider: provider,
 	})
@@ -100,6 +107,42 @@ func startFleetPing(
 	return pinger, nil
 }
 
+// fleetHealth reads how the box ran lately, for the report's health buckets:
+// the planner's last 24 hours from the plans it kept, the drivers' last 24
+// hours of polls once the count covers six of them, and the newest update of
+// the last 7 days. A source that is missing or too short, as the updater is
+// on a box that does not update itself, reads as nothing to report.
+func fleetHealth(st *state.Store, tel *telemetry.Store, updates *selfupdate.Checker, now time.Time) fleetping.Health {
+	var h fleetping.Health
+	if st != nil {
+		plans, slowestMs, fallbacks, err := st.PlannerDay(now.Add(-24 * time.Hour).UnixMilli())
+		if err != nil {
+			slog.Debug("fleet report: planner day unavailable", "err", err)
+		} else {
+			h.Plans, h.SlowestSolve, h.Fallbacks = plans, time.Duration(slowestMs*float64(time.Millisecond)), fallbacks
+		}
+	}
+	// Counts that cover under six hours, as after an update's restart, say
+	// too little: a start-up burst of failed polls would read as trouble.
+	if tel != nil {
+		if polls, errors, covered := tel.PollsLastDay(now); covered >= 6*time.Hour {
+			h.Polls, h.PollErrors = polls, errors
+		}
+	}
+	if updates != nil {
+		u := updates.Status()
+		if u.Action == "update" && !u.UpdatedAt.IsZero() && now.Sub(u.UpdatedAt) <= 7*24*time.Hour {
+			switch u.State {
+			case "done":
+				h.Update = "ok"
+			case "failed":
+				h.Update = "failed"
+			}
+		}
+	}
+	return h
+}
+
 // fleetProvider sends the report to FTW Cloud through the relay when this
 // build names one, and only then: FTW Cloud then never sees the box's
 // address. A build without a relay posts straight to /v1/fleet as before. An

go/cmd/ftw/fleet_ping_test.go

@@ -13,13 +13,17 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/srcfl/ftw/go/internal/api"
 	"github.com/srcfl/ftw/go/internal/components"
 	"github.com/srcfl/ftw/go/internal/config"
 	"github.com/srcfl/ftw/go/internal/control"
 	"github.com/srcfl/ftw/go/internal/driverrepo"
+	"github.com/srcfl/ftw/go/internal/fleetping"
+	"github.com/srcfl/ftw/go/internal/selfupdate"
 	"github.com/srcfl/ftw/go/internal/state"
+	"github.com/srcfl/ftw/go/internal/telemetry"
 )
 
 // A config saved from the Settings screen does not pass through applyDefaults:
@@ -117,7 +121,7 @@ func TestRunningFleetPingSeesADriverInstalledLater(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
 	if err != nil {
 		t.Fatalf("startFleetPing: %v", err)
 	}
@@ -339,7 +343,7 @@ func TestNoNameTheHouseholdChoseReachesThePostedPing(t *testing.T) {
 	// path unchanged. Payload() is what sendOnce marshals and posts.
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
 	if err != nil {
 		t.Fatalf("startFleetPing: %v", err)
 	}
@@ -469,7 +473,7 @@ func TestARepositoryRootInsideTheBundledTreeIsNotTheShippedCatalogue(t *testing.
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, "v1.4.0", active)
+	pinger, err := startFleetPing(ctx, cfg, cfgMu, store, statsAnswer{st: store}, nil, nil, "v1.4.0", active)
 	if err != nil {
 		t.Fatalf("startFleetPing: %v", err)
 	}
@@ -499,3 +503,53 @@ func slicesContain(haystack []string, needle string) bool {
 	}
 	return false
 }
+
+// The report's health comes from the plans the box kept, the drivers' polls
+// and the newest update, and leaves only as buckets. A box without an updater
+// reads as no update, like a box that had none.
+func TestFleetHealthReadsTheBoxsLastDay(t *testing.T) {
+	store := openStore(t)
+	now := time.Now()
+	for i, js := range []string{`{"solver":{"solve_ms":6100,"fallback":true}}`, `{"solver":{"solve_ms":300}}`} {
+		if err := store.SaveDiagnostic(now.Add(-time.Duration(i+1)*time.Hour).UnixMilli(), "scheduled", "SE3", 0, 96, js); err != nil {
+			t.Fatal(err)
+		}
+	}
+	tel := telemetry.NewStore()
+	for range 99 {
+		tel.CountPoll(false)
+	}
+	tel.CountPoll(true)
+	updates := selfupdate.New(selfupdate.Config{StatusPath: filepath.Join(t.TempDir(), "update-status.json")}, nil)
+	if err := updates.WriteStatus(selfupdate.UpdateStatus{State: "failed", Action: "update", Target: "v9.9.9", UpdatedAt: now.Add(-48 * time.Hour)}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Just after a start the poll count covers too little to say anything.
+	if h := fleetHealth(store, tel, updates, now); h.Polls != 0 || h.PollErrors != 0 || h.Plans != 2 {
+		t.Fatalf("a fresh count was reported: %+v", h)
+	}
+	// Seven hours on, it covers enough; the polls and plans are still in
+	// their 24 hours.
+	h := fleetHealth(store, tel, updates, now.Add(7*time.Hour))
+	if h.Plans != 2 || h.SlowestSolve != 6100*time.Millisecond || h.Fallbacks != 1 || h.Polls != 100 || h.PollErrors != 1 || h.Update != "failed" {
+		t.Fatalf("%+v", h)
+	}
+	p := fleetping.Build(fleetping.Facts{Health: h}, now)
+	if p.Solve != "5s+" || p.Fallbacks != "1-3" || p.Errors != "1%+" || p.Update != "failed" {
+		t.Fatalf("%+v", p)
+	}
+
+	// A week on, the failed update is no longer news; a restart is not an
+	// update at all.
+	if h := fleetHealth(store, tel, updates, now.Add(6*24*time.Hour)); h.Update != "" {
+		t.Fatalf("a week-old update still reads %q", h.Update)
+	}
+	_ = updates.WriteStatus(selfupdate.UpdateStatus{State: "done", Action: "restart", UpdatedAt: now})
+	if h := fleetHealth(store, tel, updates, now); h.Update != "" {
+		t.Fatalf("a restart reads as an update: %q", h.Update)
+	}
+	if h := fleetHealth(store, tel, nil, now); h.Update != "" || h.Plans != 2 {
+		t.Fatalf("without an updater: %+v", h)
+	}
+}

go/cmd/ftw/main.go

@@ -2495,7 +2495,7 @@ func main() {
 	migrateStatistics(cfg, cfgMu, st, cloudLink)
 	answer := statsAnswer{st: st, link: cloudLink}
 	fleetPinger, fleetPingErr := startFleetPing(
-		ctx, cfg, cfgMu, st, answer, Version, config.ManagedDriversDirOverride,
+		ctx, cfg, cfgMu, st, answer, tel, selfUpdater, Version, config.ManagedDriversDirOverride,
 	)
 	if fleetPingErr != nil {
 		slog.Warn("fleet ping unavailable", "err", fleetPingErr)

go/internal/drivers/registry.go

@@ -1110,6 +1110,9 @@ func (r *Registry) runLoop(rd *runningDriver) {
 			pollCtx, finishPoll := rd.beginCommand(ctx)
 			returned, err := rd.driver.Poll(pollCtx)
 			finishPoll()
+			if r.tel != nil {
+				r.tel.CountPoll(err != nil)
+			}
 			if err != nil {
 				slog.Warn("driver poll failed", "name", rd.cfg.Name, "err", err)
 				if r.tel != nil {

go/internal/drivers/registry_restart_test.go

@@ -1332,3 +1332,29 @@ function driver_command(action, w, cmd) end
 		t.Errorf("LastSuccess advanced without any host.emit call: %v", h.LastSuccess)
 	}
 }
+
+// The fleet report's share of failed polls counts the poll loop's polls,
+// failed or not, and nothing else.
+func TestRunLoopCountsPollsAndFailedPolls(t *testing.T) {
+	r := NewRegistry(telemetry.NewStore())
+	src := `
+local n = 0
+function driver_init(config) host.set_poll_interval(50) end
+function driver_poll()
+  n = n + 1
+  if n % 2 == 0 then error("meter did not answer") end
+  return 50
+end
+function driver_command(action, w, cmd) end
+`
+	path := writeTestDriver(t, src)
+	if err := r.Add(context.Background(), config.Driver{Name: "d1", Lua: path}); err != nil {
+		t.Fatal(err)
+	}
+	defer r.Remove("d1")
+	time.Sleep(400 * time.Millisecond)
+	polls, errors, _ := r.tel.PollsLastDay(time.Now())
+	if polls < 4 || errors < 2 || errors > polls/2+1 {
+		t.Fatalf("polls %d, failed %d after 400ms of 50ms polls, every second one failing", polls, errors)
+	}
+}

go/internal/fleetping/fleetping.go

@@ -5,6 +5,9 @@
 // on which version, with which drivers, so engineering effort lands where the
 // fleet actually is. It runs against the constraint in docs/architecture.md:
 // the daily totals must not give the relay a stable household record.
+// It also says, coarsely, how the box ran over the last day: how slow its
+// slowest plan was, whether plans fell back, how often driver polls failed and
+// whether an update failed, so the fleet's trouble is found where it is.
 //
 // So one ping is a picture of one box's shape and carries nothing that joins
 // two pings together:
@@ -130,6 +133,30 @@ type Facts struct {
 
 	// InstalledAt is when this box first ran FTW.
 	InstalledAt time.Time
+
+	// Health is how the box ran lately.
+	Health Health
+}
+
+// Health is how the box ran lately, before it is coarsened. Every field is a
+// sum over the whole box: never a driver's name, a plan's time or an
+// update's version.
+type Health struct {
+	// Plans is how many plans the planner kept in the last 24 hours,
+	// SlowestSolve the longest any of them took, and Fallbacks how many were
+	// a fallback plan.
+	Plans        int
+	SlowestSolve time.Duration
+	Fallbacks    int
+
+	// Polls is how many driver polls ran in the last 24 hours, all drivers
+	// together, and PollErrors how many failed. Both stay zero while the
+	// count covers too little of the day to say anything.
+	Polls, PollErrors uint64
+
+	// Update is the outcome of the newest update in the last 7 days: "ok",
+	// "failed", or empty without one.
+	Update string
 }
 
 // Payload is the whole ping. Every field is a bucket or a label shared with
@@ -144,14 +171,82 @@ type Payload = fleetwire.Report
 func Build(f Facts, now time.Time) Payload {
 	version, channel := releaseVersion(f.Version)
 	return Payload{
-		Schema:  Schema,
-		Version: version,
-		Channel: channel,
-		Drivers: driverTypes(f.Drivers, f.Catalogue),
-		Battery: batteryBucket(f.BatteryWh),
-		Zone:    knownZone(f.PriceZone),
-		Age:     ageBucket(f.InstalledAt, now),
+		Schema:    Schema,
+		Version:   version,
+		Channel:   channel,
+		Drivers:   driverTypes(f.Drivers, f.Catalogue),
+		Battery:   batteryBucket(f.BatteryWh),
+		Zone:      knownZone(f.PriceZone),
+		Age:       ageBucket(f.InstalledAt, now),
+		Solve:     solveBucket(f.Health),
+		Fallbacks: fallbackBucket(f.Health),
+		Errors:    errorBucket(f.Health),
+		Update:    updateBucket(f.Health.Update),
+	}
+}
+
+// The health buckets answer where engineering time goes: is the planner fast
+// enough on the hardware boxes run, does it fall back, do drivers fail, do
+// updates. Each is three or four words wide and describes the last day, or
+// for updates the last week, so it changes as the box runs instead of
+// becoming one more fixed trait of the household. Most boxes share the
+// common answer, which is what keeps the rarer ones in a crowd.
+
+// solveBucket is the slowest plan of the last 24 hours. Five seconds is the
+// edge that matters: past it a small box's planner runs out of its budget.
+func solveBucket(h Health) string {
+	switch s := h.SlowestSolve; {
+	case h.Plans <= 0:
+		return "none"
+	case s < time.Second:
+		return "0-1s"
+	case s < 5*time.Second:
+		return "1-5s"
+	default:
+		return "5s+"
+	}
+}
+
+// fallbackBucket is how many of the last 24 hours' plans fell back.
+func fallbackBucket(h Health) string {
+	switch {
+	case h.Plans <= 0:
+		return "none"
+	case h.Fallbacks <= 0:
+		return "0"
+	case h.Fallbacks <= 3:
+		return "1-3"
+	default:
+		return "4+"
+	}
+}
+
+// errorBucket is the share of the last 24 hours' driver polls that failed.
+// A share, not a count, so a driver that polls every second and one that
+// polls every minute land in the same bucket for the same trouble. None is
+// no polls, or too few hours of them, as just after a restart.
+func errorBucket(h Health) string {
+	switch {
+	case h.Polls == 0:
+		return "none"
+	case h.PollErrors == 0:
+		return "0"
+	case h.PollErrors*100 < h.Polls:
+		return "0-1%"
+	default:
+		return "1%+"
+	}
+}
+
+// updateBucket is the newest update of the last 7 days. A box that does not
+// update itself says none, like a box that had no update, so how a box was
+// installed does not become one more field.
+func updateBucket(outcome string) string {
+	switch outcome {
+	case "ok", "failed":
+		return outcome
 	}
+	return "none"
 }
 
 // releaseTag matches the two shapes this project publishes: vX.Y.Z and