diff --git a/README.md b/README.md index bda3b0b..4bdd236 100644 --- a/README.md +++ b/README.md @@ -457,8 +457,11 @@ Anonymous box reports use `POST /v1/fleet`, or, from a Core release that names a relay, `POST /v1/ohttp` (see "Anonymous daily reports" below). The direct route sees the box's address; it stays for older releases and can go one stable Core release after the first that pins the relay. Either way it -accepts only the seven `ftw.fleet/1` fields and reduces each report into -bounded daily counts for 90 days. It stores no raw report, address or home id. An operator can read +accepts only the seven `ftw.fleet/1` fields, or `ftw.fleet/2`, which adds four +health buckets: the slowest plan of the box's last day, how many plans fell +back, the share of driver polls that failed, and the newest update of its +last week. It reduces each report into bounded daily counts, one per field +and never their combination, for 90 days. It stores no raw report, address or home id. An operator can read `/fleet/stats` only on the loopback metrics listener, with `docker exec ftw-cloud-ftwcloudd-1 ftwcloudd fleet`. These counts measure reports, not unique boxes. @@ -511,7 +514,7 @@ public multi-shard totals need an explicit safe collector. The separate anonymous daily-report feed is `GET /v1/fleet/public`. It exposes only daily counts and dimension labels with at least 5 reports, for at most -90 days. It contains imported relay history and new reports received by Cloud; +90 days. It leaves the health counts out; only `/v1/ops` carries them. It contains imported relay history and new reports received by Cloud; consumers replace the old archive rather than adding both. It needs no hourly energy activation. Updated Core sends it only after the current project statistics answer; legacy versions retain their existing daily-report setting. Averaging seven complete UTC days gives an estimate of active diff --git a/go.mod b/go.mod index 9e1ade0..43f8826 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe + github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 9403679..e20c69c 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe h1:fVahq1vonhGzVCiTyW3sxZDNul8Nn0VP8YBDUHhVEms= -github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0 h1:iK/CxTZOsxEbkXT28eaX5JUx7J0u1abH4bscHpPmAYE= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/fleet/fleet_test.go b/internal/fleet/fleet_test.go index 42af44c..821556d 100644 --- a/internal/fleet/fleet_test.go +++ b/internal/fleet/fleet_test.go @@ -28,7 +28,7 @@ func TestReportsBecomeOnlyBoundedDailyTotalsAcrossARestart(t *testing.T) { } now := time.Now().UTC().Truncate(24 * time.Hour) svc.now = func() time.Time { return now } - report := fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"} + report := fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.146.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"} raw, _ := json.Marshal(report) var wg sync.WaitGroup for range 100 { diff --git a/internal/fleet/public_test.go b/internal/fleet/public_test.go index b618ddd..1b834d5 100644 --- a/internal/fleet/public_test.go +++ b/internal/fleet/public_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http/httptest" "path/filepath" + "strings" "testing" "time" @@ -23,7 +24,7 @@ func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) { } now := time.Now() svc.now = func() time.Time { return now } - report := fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.147.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"} + report := fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.147.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"} for range 4 { _, err = reg.FleetPut(report, now) if err != nil { @@ -61,3 +62,42 @@ func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) { t.Fatal("safe cell hidden") } } + +// A report that says how the box ran is taken beside the older shape. Its +// health counts stay out of the public view, which lists its fields by name. +func TestHealthIsCountedButNotPublished(t *testing.T) { + reg, err := registry.Open(filepath.Join(t.TempDir(), "db")) + if err != nil { + t.Fatal(err) + } + defer reg.Close() + svc, err := New(reg) + if err != nil { + t.Fatal(err) + } + for _, body := range []string{ + `{"schema":"ftw.fleet/1","ftw_version":"v0.147.0","channel":"stable","drivers":[],"battery_kwh":"none","price_zone":"SE4","install_age":"unknown"}`, + `{"schema":"ftw.fleet/2","ftw_version":"v0.147.0","channel":"stable","drivers":[],"battery_kwh":"none","price_zone":"SE4","install_age":"unknown","plan_solve":"5s+","plan_fallbacks":"1-3","driver_errors":"0","update":"failed"}`, + } { + w := httptest.NewRecorder() + svc.ServeHTTP(w, httptest.NewRequest("POST", "/v1/fleet", strings.NewReader(body))) + if w.Code != 204 { + t.Fatalf("%d for %s", w.Code, body) + } + } + view, err := reg.FleetView(time.Now()) + if err != nil || len(view.Days) != 1 { + t.Fatal(view, err) + } + d := view.Days[0] + if d.Reports != 2 || d.Solve["5s+"] != 1 || d.Fallbacks["1-3"] != 1 || d.Errors["0"] != 1 || d.Update["failed"] != 1 || len(d.Update) != 1 { + t.Fatalf("%+v", d) + } + w := httptest.NewRecorder() + svc.Public(w, httptest.NewRequest("GET", "/v1/fleet/public", nil)) + for _, key := range []string{"plan_solve", "plan_fallbacks", "driver_errors", `"update"`} { + if strings.Contains(w.Body.String(), key) { + t.Fatalf("the public view carries %s: %s", key, w.Body.String()) + } + } +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index cdc50b0..f641393 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -152,7 +152,7 @@ func (s *Service) add(msg []byte) int { var added, repeat bool var err error switch reportwire.Schema(report.Body) { - case fleetwire.Schema: + case fleetwire.Schema, fleetwire.SchemaV1: fleet, decodeErr := fleetwire.Decode(report.Body) if decodeErr != nil { return http.StatusBadRequest diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index e7ff290..fe8a785 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -129,9 +129,20 @@ func TestAFleetReportAddsToTheFleetTotals(t *testing.T) { } var r fleetwire.Report _ = json.Unmarshal([]byte(fleet), &r) - if r.Schema != fleetwire.Schema { + if r.Schema != fleetwire.SchemaV1 { t.Fatal("fixture") } + // The report that says how the box ran takes the same way, and its + // health is counted beside the shape. + withHealth := strings.Replace(strings.TrimSuffix(fleet, "}"), `"ftw.fleet/1"`, `"ftw.fleet/2"`, 1) + + `,"plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"failed"}` + if got := send(t, s, cfg, inner(withHealth).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("status %d for an ftw.fleet/2 report", got) + } + view, err = reg.FleetView(now) + if err != nil || view.Days[0].Reports != 2 || view.Days[0].Update["failed"] != 1 || view.Days[0].Solve["1-5s"] != 1 { + t.Fatalf("fleet %+v %v", view, err) + } } // Anything but the one padded request with one valid report gets a sealed diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go index eabfbc9..cd78237 100644 --- a/internal/ops/ops_test.go +++ b/internal/ops/ops_test.go @@ -346,3 +346,35 @@ func keys(m map[string]json.RawMessage) []string { slices.Sort(out) return out } + +// The health counts of reports that carry them come exact in the feed's +// closed fleet days; a day without any has no health fields. +func TestTheFeedCarriesHealthCounts(t *testing.T) { + s, reg := newService(t, hash(key)) + yesterday := now.UTC().AddDate(0, 0, -1) + for _, raw := range []string{ + `{"schema":"ftw.fleet/2","ftw_version":"v0.155.1","channel":"stable","drivers":["sungrow"],"battery_kwh":"5-15","price_zone":"SE3","install_age":"1-6m","plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"ok"}`, + `{"schema":"ftw.fleet/1","ftw_version":"v0.155.1","channel":"stable","drivers":["sungrow"],"battery_kwh":"5-15","price_zone":"SE3","install_age":"1-6m"}`, + } { + r, err := fleetwire.Decode([]byte(raw)) + if err != nil { + t.Fatal(err) + } + if ok, err := reg.FleetPut(r, yesterday); !ok || err != nil { + t.Fatal(ok, err) + } + } + if ok, err := reg.FleetPut(fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "unknown", Channel: "unknown", Drivers: []string{}, Battery: "none", Zone: "unknown", Age: "unknown"}, yesterday.AddDate(0, 0, -1)); !ok || err != nil { + t.Fatal(ok, err) + } + var f struct { + Days []map[string]json.RawMessage `json:"fleet_days"` + } + _ = json.Unmarshal(get(s, Path, "Bearer "+key).Body.Bytes(), &f) + if len(f.Days) != 2 || len(f.Days[0]) != 12 || string(f.Days[0]["update"]) != `{"ok":1}` || string(f.Days[0]["plan_solve"]) != `{"1-5s":1}` { + t.Fatalf("the newest day: %v", f.Days) + } + if _, ok := f.Days[1]["plan_solve"]; ok || len(f.Days[1]) != 8 { + t.Fatalf("a day of older reports has health fields: %v", f.Days[1]) + } +} diff --git a/internal/server/reports_e2e_test.go b/internal/server/reports_e2e_test.go index 20ed7d5..bf9020e 100644 --- a/internal/server/reports_e2e_test.go +++ b/internal/server/reports_e2e_test.go @@ -70,7 +70,7 @@ func TestBothReportsReachTheirTotalsThroughARelay(t *testing.T) { yesterday := time.Now().UTC().AddDate(0, 0, -1).Format(time.DateOnly) report := energywire.Report{Schema: energywire.Schema, Date: yesterday, PVGeneration: 12, GridImport: 30, GridExport: 4, BatteryCharge: 9, BatteryDischarge: 8} energyBody, _ := json.Marshal(report) - fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"}) + fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"}) // Three energy reports and two fleet reports, each alone. for i, body := range [][]byte{energyBody, fleetBody, energyBody, fleetBody, energyBody} { id := make([]byte, reportwire.IDBytes) diff --git a/privacy/internal-operations.md b/privacy/internal-operations.md index 8250458..3f359e2 100644 --- a/privacy/internal-operations.md +++ b/privacy/internal-operations.md @@ -28,6 +28,13 @@ utfall och deras tid, handskakningar, dräneringar och Go-körtiden. En flottdag visas när den är slut, ett energidatum när ingen rapport längre får nämna det. Därför skiljer sig två hämtningar aldrig med en enda rapport. +Rapporter i formatet `ftw.fleet/2` säger också grovt hur boxen gick: den +långsammaste planen och antalet reservplaner under boxens senaste dygn, +andelen misslyckade drivrutinsläsningar och utfallet av den senaste +uppdateringen den senaste veckan. Varje mått är några fasta ord. Cloud räknar +dem per dag och fält, aldrig i kombination, och den offentliga vyn visar dem +inte. Argumentet för att de inte pekar ut en box står i srcfl/ftw#1821. + ## Vem ser den Bara den som Cloudflare Access släpper in på `stats.ftw.energy/admin*` och