Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -457,8 +457,11 @@ Anonymous box reports use `POST /v1/fleet`, or, from a Core release that
names a relay, `POST /v1/ohttp` (see "Anonymous daily reports" below). The
direct route sees the box's address; it stays for older releases and can go
one stable Core release after the first that pins the relay. Either way it
accepts only the seven `ftw.fleet/1` fields and reduces each report into
bounded daily counts for 90 days. It stores no raw report, address or home id. An operator can read
accepts only the seven `ftw.fleet/1` fields, or `ftw.fleet/2`, which adds four
health buckets: the slowest plan of the box's last day, how many plans fell
back, the share of driver polls that failed, and the newest update of its
last week. It reduces each report into bounded daily counts, one per field
and never their combination, for 90 days. It stores no raw report, address or home id. An operator can read
`/fleet/stats` only on the loopback metrics listener, with
`docker exec ftw-cloud-ftwcloudd-1 ftwcloudd fleet`. These counts measure
reports, not unique boxes.
Expand Down Expand Up @@ -511,7 +514,7 @@ public multi-shard totals need an explicit safe collector.

The separate anonymous daily-report feed is `GET /v1/fleet/public`. It exposes
only daily counts and dimension labels with at least 5 reports, for at most
90 days. It contains imported relay history and new reports received by Cloud;
90 days. It leaves the health counts out; only `/v1/ops` carries them. It contains imported relay history and new reports received by Cloud;
consumers replace the old archive rather than adding both. It needs no hourly
energy activation. Updated Core sends it only after the current project
statistics answer; legacy versions retain their existing daily-report setting. Averaging seven complete UTC days gives an estimate of active
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/fxamacker/cbor/v2 v2.9.4
github.com/gorilla/websocket v1.5.3
github.com/quic-go/quic-go v0.63.0
github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe
github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0
golang.org/x/crypto v0.57.0
golang.org/x/net v0.58.0
golang.org/x/sys v0.48.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78
github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe h1:fVahq1vonhGzVCiTyW3sxZDNul8Nn0VP8YBDUHhVEms=
github.com/srcfl/ftw/go/protocol v0.0.0-20261024082233-5cf4a80689fe/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0 h1:iK/CxTZOsxEbkXT28eaX5JUx7J0u1abH4bscHpPmAYE=
github.com/srcfl/ftw/go/protocol v0.0.0-20261024093922-2d7c8eba41f0/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
Expand Down
2 changes: 1 addition & 1 deletion internal/fleet/fleet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestReportsBecomeOnlyBoundedDailyTotalsAcrossARestart(t *testing.T) {
}
now := time.Now().UTC().Truncate(24 * time.Hour)
svc.now = func() time.Time { return now }
report := fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"}
report := fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.146.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"}
raw, _ := json.Marshal(report)
var wg sync.WaitGroup
for range 100 {
Expand Down
42 changes: 41 additions & 1 deletion internal/fleet/public_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"

Expand All @@ -23,7 +24,7 @@ func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) {
}
now := time.Now()
svc.now = func() time.Time { return now }
report := fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.147.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"}
report := fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.147.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"}
for range 4 {
_, err = reg.FleetPut(report, now)
if err != nil {
Expand Down Expand Up @@ -61,3 +62,42 @@ func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) {
t.Fatal("safe cell hidden")
}
}

// A report that says how the box ran is taken beside the older shape. Its
// health counts stay out of the public view, which lists its fields by name.
func TestHealthIsCountedButNotPublished(t *testing.T) {
reg, err := registry.Open(filepath.Join(t.TempDir(), "db"))
if err != nil {
t.Fatal(err)
}
defer reg.Close()
svc, err := New(reg)
if err != nil {
t.Fatal(err)
}
for _, body := range []string{
`{"schema":"ftw.fleet/1","ftw_version":"v0.147.0","channel":"stable","drivers":[],"battery_kwh":"none","price_zone":"SE4","install_age":"unknown"}`,
`{"schema":"ftw.fleet/2","ftw_version":"v0.147.0","channel":"stable","drivers":[],"battery_kwh":"none","price_zone":"SE4","install_age":"unknown","plan_solve":"5s+","plan_fallbacks":"1-3","driver_errors":"0","update":"failed"}`,
} {
w := httptest.NewRecorder()
svc.ServeHTTP(w, httptest.NewRequest("POST", "/v1/fleet", strings.NewReader(body)))
if w.Code != 204 {
t.Fatalf("%d for %s", w.Code, body)
}
}
view, err := reg.FleetView(time.Now())
if err != nil || len(view.Days) != 1 {
t.Fatal(view, err)
}
d := view.Days[0]
if d.Reports != 2 || d.Solve["5s+"] != 1 || d.Fallbacks["1-3"] != 1 || d.Errors["0"] != 1 || d.Update["failed"] != 1 || len(d.Update) != 1 {
t.Fatalf("%+v", d)
}
w := httptest.NewRecorder()
svc.Public(w, httptest.NewRequest("GET", "/v1/fleet/public", nil))
for _, key := range []string{"plan_solve", "plan_fallbacks", "driver_errors", `"update"`} {
if strings.Contains(w.Body.String(), key) {
t.Fatalf("the public view carries %s: %s", key, w.Body.String())
}
}
}
2 changes: 1 addition & 1 deletion internal/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func (s *Service) add(msg []byte) int {
var added, repeat bool
var err error
switch reportwire.Schema(report.Body) {
case fleetwire.Schema:
case fleetwire.Schema, fleetwire.SchemaV1:
fleet, decodeErr := fleetwire.Decode(report.Body)
if decodeErr != nil {
return http.StatusBadRequest
Expand Down
13 changes: 12 additions & 1 deletion internal/gateway/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,20 @@ func TestAFleetReportAddsToTheFleetTotals(t *testing.T) {
}
var r fleetwire.Report
_ = json.Unmarshal([]byte(fleet), &r)
if r.Schema != fleetwire.Schema {
if r.Schema != fleetwire.SchemaV1 {
t.Fatal("fixture")
}
// The report that says how the box ran takes the same way, and its
// health is counted beside the shape.
withHealth := strings.Replace(strings.TrimSuffix(fleet, "}"), `"ftw.fleet/1"`, `"ftw.fleet/2"`, 1) +
`,"plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"failed"}`
if got := send(t, s, cfg, inner(withHealth).Marshal(reportwire.RequestSize)); got != http.StatusNoContent {
t.Fatalf("status %d for an ftw.fleet/2 report", got)
}
view, err = reg.FleetView(now)
if err != nil || view.Days[0].Reports != 2 || view.Days[0].Update["failed"] != 1 || view.Days[0].Solve["1-5s"] != 1 {
t.Fatalf("fleet %+v %v", view, err)
}
}

// Anything but the one padded request with one valid report gets a sealed
Expand Down
32 changes: 32 additions & 0 deletions internal/ops/ops_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -346,3 +346,35 @@ func keys(m map[string]json.RawMessage) []string {
slices.Sort(out)
return out
}

// The health counts of reports that carry them come exact in the feed's
// closed fleet days; a day without any has no health fields.
func TestTheFeedCarriesHealthCounts(t *testing.T) {
s, reg := newService(t, hash(key))
yesterday := now.UTC().AddDate(0, 0, -1)
for _, raw := range []string{
`{"schema":"ftw.fleet/2","ftw_version":"v0.155.1","channel":"stable","drivers":["sungrow"],"battery_kwh":"5-15","price_zone":"SE3","install_age":"1-6m","plan_solve":"1-5s","plan_fallbacks":"0","driver_errors":"0-1%","update":"ok"}`,
`{"schema":"ftw.fleet/1","ftw_version":"v0.155.1","channel":"stable","drivers":["sungrow"],"battery_kwh":"5-15","price_zone":"SE3","install_age":"1-6m"}`,
} {
r, err := fleetwire.Decode([]byte(raw))
if err != nil {
t.Fatal(err)
}
if ok, err := reg.FleetPut(r, yesterday); !ok || err != nil {
t.Fatal(ok, err)
}
}
if ok, err := reg.FleetPut(fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "unknown", Channel: "unknown", Drivers: []string{}, Battery: "none", Zone: "unknown", Age: "unknown"}, yesterday.AddDate(0, 0, -1)); !ok || err != nil {
t.Fatal(ok, err)
}
var f struct {
Days []map[string]json.RawMessage `json:"fleet_days"`
}
_ = json.Unmarshal(get(s, Path, "Bearer "+key).Body.Bytes(), &f)
if len(f.Days) != 2 || len(f.Days[0]) != 12 || string(f.Days[0]["update"]) != `{"ok":1}` || string(f.Days[0]["plan_solve"]) != `{"1-5s":1}` {
t.Fatalf("the newest day: %v", f.Days)
}
if _, ok := f.Days[1]["plan_solve"]; ok || len(f.Days[1]) != 8 {
t.Fatalf("a day of older reports has health fields: %v", f.Days[1])
}
}
2 changes: 1 addition & 1 deletion internal/server/reports_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func TestBothReportsReachTheirTotalsThroughARelay(t *testing.T) {
yesterday := time.Now().UTC().AddDate(0, 0, -1).Format(time.DateOnly)
report := energywire.Report{Schema: energywire.Schema, Date: yesterday, PVGeneration: 12, GridImport: 30, GridExport: 4, BatteryCharge: 9, BatteryDischarge: 8}
energyBody, _ := json.Marshal(report)
fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"})
fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.SchemaV1, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"})
// Three energy reports and two fleet reports, each alone.
for i, body := range [][]byte{energyBody, fleetBody, energyBody, fleetBody, energyBody} {
id := make([]byte, reportwire.IDBytes)
Expand Down
7 changes: 7 additions & 0 deletions privacy/internal-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ utfall och deras tid, handskakningar, dräneringar och Go-körtiden.
En flottdag visas när den är slut, ett energidatum när ingen rapport längre
får nämna det. Därför skiljer sig två hämtningar aldrig med en enda rapport.

Rapporter i formatet `ftw.fleet/2` säger också grovt hur boxen gick: den
långsammaste planen och antalet reservplaner under boxens senaste dygn,
andelen misslyckade drivrutinsläsningar och utfallet av den senaste
uppdateringen den senaste veckan. Varje mått är några fasta ord. Cloud räknar
dem per dag och fält, aldrig i kombination, och den offentliga vyn visar dem
inte. Argumentet för att de inte pekar ut en box står i srcfl/ftw#1821.

## Vem ser den

Bara den som Cloudflare Access släpper in på `stats.ftw.energy/admin*` och
Expand Down
Loading