From 43c77a083734af31b01559b4b028427f6a970d07 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 07:58:23 +0200 Subject: [PATCH 1/8] feat: an Oblivious HTTP gateway for anonymous daily energy reports ftwcloudd is the gateway (RFC 9458) for the box's daily energy report, which reaches it through an independent relay, so the service sees the report and never the box's address. - POST /v1/ohttp opens a request with the gateway key, takes only the one padded inner request with one valid report (energywire), adds its six sums to the date's row and answers a sealed 204 or 400. Nothing is kept per report: no row, logged body or address. One shared rate budget, and at most 100,000 reports a date. - GET /.well-known/ohttp-gateway serves the key configuration. - GET /v1/energy/public lists the closed dates of the last 30 days; a date with fewer than 5 reports shows null. - The key comes from Parameter Store into tmpfs at boot, as the Noise key does (deploy/ohttp.md, deploy/ohttp.sh); keygen -ohttp prints the pin Core needs. Without a key the gateway routes answer 501. Pairs with srcfl/ftw feat/anonymous-energy-ohttp; go/protocol is pinned at its commit 28fddd802242. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 43 +++++ cmd/ftwcloudd/main.go | 24 ++- cmd/ftwcloudd/ohttp.go | 100 ++++++++++++ cmd/ftwcloudd/ohttp_test.go | 93 +++++++++++ deploy/README.md | 5 +- deploy/bootstrap.sh | 2 +- deploy/compose.yml | 5 + deploy/ohttp.md | 109 +++++++++++++ deploy/ohttp.sh | 67 ++++++++ go.mod | 2 +- go.sum | 2 + internal/energy/energy.go | 198 +++++++++++++++++++++++ internal/energy/energy_test.go | 249 +++++++++++++++++++++++++++++ internal/registry/energy.go | 79 +++++++++ internal/registry/energy_test.go | 57 +++++++ internal/server/energy_e2e_test.go | 103 ++++++++++++ internal/server/server.go | 20 ++- internal/server/server_test.go | 4 + 19 files changed, 1156 insertions(+), 8 deletions(-) create mode 100644 cmd/ftwcloudd/ohttp.go create mode 100644 cmd/ftwcloudd/ohttp_test.go create mode 100644 deploy/ohttp.md create mode 100755 deploy/ohttp.sh create mode 100644 internal/energy/energy.go create mode 100644 internal/energy/energy_test.go create mode 100644 internal/registry/energy.go create mode 100644 internal/registry/energy_test.go create mode 100644 internal/server/energy_e2e_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 04b8019..184a269 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ maintains it. grant serves a copy; SQLite holds only the home id, the box key, the signed roster, owners' suspensions, tombstones, APNs registrations, the hashes of extension tokens when each away box's alert is due, browser notice sent markers and - anonymous daily fleet totals. Browser endpoints and box-encrypted offline + anonymous daily fleet and energy totals. Browser endpoints and box-encrypted offline notices live only in the signed roster. Nothing logs addresses. - **A phone can only narrow, or talk to its own box.** On `/v1/app` it can diff --git a/README.md b/README.md index 3358fca..210cdad 100644 --- a/README.md +++ b/README.md @@ -512,3 +512,46 @@ consumers replace the old archive rather than adding both. It needs no hourly energy activation. Updated Core sends it only after the current project statistics answer; legacy versions retain their existing daily-report setting. Averaging seven complete UTC days gives an estimate of active reporting installations, never people or all installations ever made. + +## Anonymous daily energy + +Energy on stats.ftw.energy comes from a separate report that is not personal +data to this service, so it needs no C8 activation. Once per closed UTC day, +1 to 12 hours after it closes, a box sends the date and six whole-kWh sums: +solar generation, grid import and export, battery charge and discharge, and +vehicle charge. Nothing else, and no field the fleet report carries. A day +with a gap in the box's energy ledger is skipped. Core sends it only under +the fleet report's answer and never for a site a partner controls +(`go/internal/energyreport` in srcfl/ftw). + +The box seals the report with Oblivious HTTP (RFC 9458) to the gateway key +its release pins, and posts it to an independent relay +([deploy/ohttp.md](deploy/ohttp.md)). The relay sees the box's address and a +ciphertext of fixed length. `POST /v1/ohttp` (`internal/energy`) sees the +report and the relay's address. It answers a sealed 204, or 400 for anything +but the one padded request with one valid report: exact fields, at most +1,000 kWh each, a date in the last three closed days. It adds the sums to +that date's row and keeps nothing per report: no row, logged body or +address. All requests share one rate budget, and a date takes at most +100,000 reports. `GET /.well-known/ohttp-gateway` serves the key +configuration for the relay operator and for anyone checking the pin. + +`GET /v1/energy/public` lists the closed dates of the last 30 days, newest +first: the report count and the six sums. A date appears only once no report +may name it any more, so two readings never differ by one report. A date with +fewer than 5 reports shows null, as `/v1/fleet/public` does. Rows go after 30 +days. These are reports, not homes. + +Limits, plainly. A report cannot prove that a real box sent it, and the +relay could replay one it forwarded; the bounds and the cap keep both to +noise. Privacy Pass-style tokens are the later fix and are not built. The +relay and Sourceful together could join reports to addresses by time. This +service also holds readings from homes with the readable copy on, including +48 hours of history tiles, from which the same daily sums could be computed; +the gateway does not do this and keeps nothing to match with, but that is a +rule in code, not one the cryptography enforces. + +Open for Fredrik: whether reports that are anonymous to Sourceful still need +the owner's yes under the ePrivacy rule on reading terminal equipment +(Swedish LEK 9:28). The design keeps the existing yes, so it is covered +either way. diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 46a2b01..7712a1f 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -3,6 +3,7 @@ // reads and never controls. See README.md. // // ftwcloudd keygen -out service.key make the service key; prints its public half +// ftwcloudd keygen -ohttp -out ohttp.key make a gateway key; prints the configuration Core pins // ftwcloudd keygen -noise -out noise.key make the Noise key phones dial; prints its public half // ftwcloudd metrics print the running service's counts, from inside its container // ftwcloudd health https://host/healthz check /healthz over TCP and HTTP/3 @@ -37,6 +38,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/apns" "github.com/srcfl/ftw-cloud/internal/appsession" + "github.com/srcfl/ftw-cloud/internal/energy" "github.com/srcfl/ftw-cloud/internal/fleet" "github.com/srcfl/ftw-cloud/internal/keys" "github.com/srcfl/ftw-cloud/internal/mcp" @@ -89,6 +91,7 @@ func serve(args []string) error { stateDir := fs.String("state", "/state", "directory for the registry and certificates") keyPath := fs.String("service-key", "/run/secrets/ftw-cloud-service.key", "service key, PEM") noisePath := fs.String("noise-key", "/run/secrets/ftw-cloud-noise.key", "Noise key phones dial, PEM; without it /v1/app is off") + ohttpPath := fs.String("ohttp-key", "/run/secrets/ftw-cloud-ohttp.key", "Oblivious HTTP gateway keys, PEM; without them /v1/ohttp is off") certFile := fs.String("tls-cert", "", "certificate file, instead of ACME (local tests)") keyFile := fs.String("tls-key", "", "certificate key file, with -tls-cert") email := fs.String("acme-email", "", "contact for the certificate authority") @@ -122,6 +125,14 @@ func serve(args []string) error { } else if err != nil { return err } + // The energy reports' gateway takes its keys the same way. Without them + // boxes send nothing that reaches here, and the public totals still answer. + gateway, err := loadOHTTPKeys(*ohttpPath) + if errors.Is(err, os.ErrNotExist) { + log.Warn("no OHTTP key: the energy report gateway is off until one is installed", "path", *ohttpPath) + } else if err != nil { + return err + } if *statsPolicy != "" && *statsPolicy != cloudwire.StatsPolicy { return fmt.Errorf("unknown statistics policy") } @@ -164,9 +175,13 @@ func serve(args []string) error { if err != nil { return err } + energySvc, err := energy.New(reg, gateway) + if err != nil { + return err + } tokens := token.New(st, reg, log) tokens.Shards = shards - srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Statistics: usageSvc, Log: log} + srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Energy: energySvc, Statistics: usageSvc, Log: log} // Assistants: the authorization server, with agent keys in ftwkeysd. agentKeys := keys.Dial(*keysSocket) @@ -386,7 +401,8 @@ func serve(args []string) error { pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) log.Info("ftwcloudd listening", "addr", *listen, "h3", srv.AltSvc != "", "quic_sessions", door != nil && door.Box != nil, "shard", shards.Self(), "shards", shards.Shards(), - "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public())) + "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public()), + "ohttp_keys", ohttpPins(gateway)) err = httpSrv.ListenAndServeTLS("", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { return err @@ -549,10 +565,14 @@ func keygen(args []string) error { fs := flag.NewFlagSet("keygen", flag.ExitOnError) out := fs.String("out", "", "file to write the key to") noise := fs.Bool("noise", false, "make the X25519 key phones dial instead of the P-256 service key") + gatewayKey := fs.Bool("ohttp", false, "make an Oblivious HTTP gateway key; prints the key configuration Core pins") _ = fs.Parse(args) if *out == "" { return errors.New("keygen: -out is required") } + if *gatewayKey { + return keygenOHTTP(*out) + } if *noise { x, err := ecdh.X25519().GenerateKey(rand.Reader) if err != nil { diff --git a/cmd/ftwcloudd/ohttp.go b/cmd/ftwcloudd/ohttp.go new file mode 100644 index 0000000..7bacd36 --- /dev/null +++ b/cmd/ftwcloudd/ohttp.go @@ -0,0 +1,100 @@ +package main + +import ( + "crypto/ecdh" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "os" + "strings" + + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// The Oblivious HTTP gateway's keys (deploy/ohttp.md). Core pins the public +// half of each as a key configuration, so a key here that no release pins +// opens nothing. + +// ohttpKeyID is the first byte of SHA-256 of the public key, so a key's id +// needs no second file. +func ohttpKeyID(pub []byte) uint8 { + sum := sha256.Sum256(pub) + return sum[0] +} + +// loadOHTTPKeys reads the gateway's X25519 keys: PKCS #8 PEM, one block per +// key. A second block lets a key rotate across a Core release. +func loadOHTTPKeys(path string) (*ohttp.Gateway, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("ohttp key: %w", err) + } + var keys []ohttp.Key + for { + var block *pem.Block + block, raw = pem.Decode(raw) + if block == nil { + break + } + if block.Type != "PRIVATE KEY" { + return nil, fmt.Errorf("ohttp key: %s holds a %s, not a PKCS #8 private key", path, block.Type) + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("ohttp key: %w", err) + } + x, ok := parsed.(*ecdh.PrivateKey) + if !ok || x.Curve() != ecdh.X25519() { + return nil, fmt.Errorf("ohttp key: %s holds a key that is not X25519", path) + } + keys = append(keys, ohttp.Key{ID: ohttpKeyID(x.PublicKey().Bytes()), Private: x}) + } + if len(keys) == 0 { + return nil, fmt.Errorf("ohttp key: %s holds no key", path) + } + gw, err := ohttp.NewGateway(keys...) + if err != nil { + return nil, fmt.Errorf("ohttp key: %s holds two keys under one id; make a new one", path) + } + return gw, nil +} + +// ohttpPins is each key configuration as Core pins it, comma separated, for +// the start line. Empty when the gateway is off. +func ohttpPins(gw *ohttp.Gateway) string { + if gw == nil { + return "" + } + var pins []string + for _, c := range gw.Configs() { + pins = append(pins, base64.RawURLEncoding.EncodeToString(c.Marshal())) + } + return strings.Join(pins, ",") +} + +// keygenOHTTP writes a new gateway key, never over an existing file, and +// prints its configuration: the string energyreport.GatewayKey in srcfl/ftw +// pins. For a rotation, put the new block after the old one in the same +// parameter; ftwcloudd refuses two keys under one id, and then a new key is +// needed. +func keygenOHTTP(path string) error { + x, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + return err + } + der, err := x509.MarshalPKCS8PrivateKey(x) + if err != nil { + return err + } + if err := writePEM(path, "PRIVATE KEY", der); err != nil { + return err + } + gw, err := ohttp.NewGateway(ohttp.Key{ID: ohttpKeyID(x.PublicKey().Bytes()), Private: x}) + if err != nil { + return err + } + fmt.Println(ohttpPins(gw)) + return nil +} diff --git a/cmd/ftwcloudd/ohttp_test.go b/cmd/ftwcloudd/ohttp_test.go new file mode 100644 index 0000000..6126066 --- /dev/null +++ b/cmd/ftwcloudd/ohttp_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "bytes" + "encoding/base64" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +// keygen prints the configuration Core pins, and ftwcloudd loads the same +// key back from the file under the same id. +func TestAGatewayKeyLoadsAsThePinSays(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ohttp.key") + pin := captureStdout(t, func() error { return keygenOHTTP(path) }) + raw, err := base64.RawURLEncoding.DecodeString(strings.TrimSpace(pin)) + if err != nil { + t.Fatal(err) + } + cfg, err := ohttp.ParseKeyConfig(raw) + if err != nil { + t.Fatal(err) + } + gw, err := loadOHTTPKeys(path) + if err != nil { + t.Fatal(err) + } + if got := gw.Configs(); len(got) != 1 || !bytes.Equal(got[0].Marshal(), raw) || cfg.ID != ohttpKeyID(cfg.PublicKey) { + t.Fatalf("loaded %+v, pinned %+v", got, cfg) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Fatalf("key file mode %v", info.Mode()) + } + if err := keygenOHTTP(path); err == nil { + t.Fatal("keygen wrote over a key") + } + + // A rotation is a second block in the same file. + second := filepath.Join(dir, "second.key") + captureStdout(t, func() error { return keygenOHTTP(second) }) + a, _ := os.ReadFile(path) + b, _ := os.ReadFile(second) + both := filepath.Join(dir, "both.key") + if err := os.WriteFile(both, append(a, b...), 0o600); err != nil { + t.Fatal(err) + } + if gw, err := loadOHTTPKeys(both); err != nil && !strings.Contains(err.Error(), "one id") || err == nil && len(gw.Configs()) != 2 { + t.Fatalf("two keys: %v", err) + } + if pins := ohttpPins(nil); pins != "" { + t.Fatal(pins) + } +} + +func TestOnlyX25519GatewayKeysLoad(t *testing.T) { + dir := t.TempDir() + if _, err := loadOHTTPKeys(filepath.Join(dir, "missing.key")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing: %v", err) + } + service := filepath.Join(dir, "service.key") + captureStdout(t, func() error { return keygen([]string{"-out", service}) }) + empty := filepath.Join(dir, "empty.key") + _ = os.WriteFile(empty, []byte("no key here\n"), 0o600) + for _, p := range []string{service, empty} { + if _, err := loadOHTTPKeys(p); err == nil { + t.Fatalf("%s loaded", p) + } + } +} + +func captureStdout(t *testing.T, f func() error) string { + t.Helper() + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + runErr := f() + os.Stdout = old + _ = w.Close() + out, _ := io.ReadAll(r) + if runErr != nil { + t.Fatal(runErr) + } + return string(out) +} diff --git a/deploy/README.md b/deploy/README.md index c1c603f..14d6655 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -381,8 +381,9 @@ Cloud data; the images on `ghcr.io` hold only code. ## Not here yet -The APNs key ([apns.md](apns.md)) and the Noise key phones dial -([noise.md](noise.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the +The APNs key ([apns.md](apns.md)), the Noise key phones dial +([noise.md](noise.md)) and the energy reports' gateway key +([ohttp.md](ohttp.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the security group ([http3.md](http3.md)). Widget tokens need no secret: the registry holds their hashes. Statistics (C8) have their own [review](../privacy/project-statistics-dpia.md) and diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index b2d1c8e..fbf1ee9 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -131,7 +131,7 @@ if ! [[ $user =~ ^[0-9]+:[0-9]+$ ]]; then exit 1 fi vol=$(docker volume inspect -f '{{.Mountpoint}}' ftw-cloud_ftw-cloud-state 2>/dev/null || true) -for p in "$vol" /run/ftw-cloud/service.key /run/ftw-cloud/apns /run/ftw-cloud/noise; do +for p in "$vol" /run/ftw-cloud/service.key /run/ftw-cloud/apns /run/ftw-cloud/noise /run/ftw-cloud/ohttp; do if [ -n "$p" ] && [ -e "$p" ]; then chown -R "$user" "$p" fi diff --git a/deploy/compose.yml b/deploy/compose.yml index e936d46..421d63b 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -28,6 +28,7 @@ services: - -service-key=/run/secrets/ftw-cloud-service.key - -acme-email=${FTW_CLOUD_ACME_EMAIL:-} - -noise-key=/run/secrets/noise/noise.key + - -ohttp-key=/run/secrets/ohttp/ohttp.key - -keys-socket=/run/ftw-cloud-keys/keys.sock environment: # /source publishes it, so anyone can compare it with what CI built. @@ -50,6 +51,10 @@ services: # boot by deploy/noise.sh. Empty means /v1/app is off, and the # service runs. - /run/ftw-cloud/noise:/run/secrets/noise:ro + # The energy reports' Oblivious HTTP gateway key, read from Parameter + # Store into tmpfs at boot by deploy/ohttp.sh. Empty means /v1/ohttp + # is off, and the service runs. + - /run/ftw-cloud/ohttp:/run/secrets/ohttp:ro # The keys part's socket (deploy/keys.md). Without ftwkeysd it holds # nothing, and assistants cannot connect. - /run/ftw-cloud-keys:/run/ftw-cloud-keys diff --git a/deploy/ohttp.md b/deploy/ohttp.md new file mode 100644 index 0000000..7874666 --- /dev/null +++ b/deploy/ohttp.md @@ -0,0 +1,109 @@ +# The energy reports' gateway key and relay + +A box sends one anonymous energy report a day, sealed with Oblivious HTTP +(RFC 9458) to this service's gateway key and posted to a relay that someone +other than Sourceful runs. The relay forwards it to `POST /v1/ohttp`. The +relay sees the box's address but not the report; this service sees the +report but not the address. Without the key, `/v1/ohttp` answers 501 and the +public totals still answer. + +The private key lives like the Noise key ([noise.md](noise.md)): made on a +machine signed in to the account, put straight into Parameter Store, and +read into tmpfs at every boot. It is never in the repository, the image, the +volume or a log. A Core release pins the public half, so boxes never ask +this service for a key. + +## 1. Make the key (once) + +From a checkout of this repository: + +```bash +go run ./cmd/ftwcloudd keygen -ohttp -out /tmp/ftw-cloud-ohttp.key +``` + +It writes the private key and prints the key configuration, base64url, 55 +characters. Keep it: it is the pin for Core in step 5, and the relay +operator may ask for it. + +## 2. Put it in Parameter Store (once) + +In a terminal signed in to the FTW Cloud account (976075258475): + +```bash +aws ssm put-parameter --region eu-north-1 --name /ftw-cloud/ohttp-key --type SecureString --value file:///tmp/ftw-cloud-ohttp.key +rm /tmp/ftw-cloud-ohttp.key +``` + +## 3. Give it to the host (once per host) + +`deploy/ohttp.sh` installs `ftw-cloud-ohttp.service`, which reads the +parameter into `/run/ftw-cloud/ohttp/` before Docker starts. `compose.yml` +mounts that directory read-only. Run it through SSM, then redeploy: + +```bash +aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ + --document-name AWS-RunShellScript \ + --parameters 'commands=["bash /opt/ftw-cloud/deploy/ohttp.sh","/usr/local/sbin/ftw-cloud-deploy"]' +``` + +The start line, `ftwcloudd listening … ohttp_keys=…`, shows the +configuration from step 1. From anywhere: + +```bash +curl -s https://cloud.ftw.energy/.well-known/ohttp-gateway | base64 +curl -s -o /dev/null -w '%{http_code}\n' -H 'Content-Type: message/ohttp-req' \ + --data-binary 'not a report' https://cloud.ftw.energy/v1/ohttp +``` + +The first prints the configuration with a two-byte length in front. The +second prints 400, because the bytes do not open under the key; without the +key it prints 501. + +## 4. Ask a relay operator (once) + +The relay must be run by someone other than Sourceful. Two that sell one: + +- **Cloudflare OHTTP Relay** (formerly Privacy Gateway). Enterprise only, in + closed beta: ask through https://www.cloudflare.com/lp/privacy-edge/. + Sourceful already uses Cloudflare, so this is the shorter path. +- **Fastly OHTTP Relay**, through Fastly sales. + +Send them: + +- **Gateway URL:** `https://cloud.ftw.energy/v1/ohttp` +- **Key configuration:** `https://cloud.ftw.energy/.well-known/ohttp-gateway` + (`application/ohttp-keys`), or the string from step 1. The suite is + DHKEM(X25519, HKDF-SHA256), HKDF-SHA256, AES-128-GCM. +- **Traffic:** `POST` with `message/ohttp-req`, always 375 bytes; answers + are `message/ohttp-res`, always 48 bytes. One request per box per day, + spread over the 12 hours after 00:00 UTC. + +Ask them to confirm, in writing: + +- they pass the gateway no client address or other client header + (`Forwarded`, `X-Forwarded-For`, `CF-Connecting-IP`, `True-Client-IP`, + `User-Agent`), only the body and its content type; +- what they log about the client and for how long. Cloudflare's published + terms keep the address, the time and request metadata for about 124 days; +- that they keep no request body. + +They hand back the **relay URL** the boxes post to. Cloudflare's form is +`https://privacy-relay.cloudflare.com/`. + +## 5. Pin both in Core + +In srcfl/ftw, `go/internal/energyreport/energyreport.go`: `RelayURL` takes the +relay URL from step 4 and `GatewayKey` the string from step 1. Boxes on the +release that carries both start sending; older releases send nothing. + +## Rotating + +Make a new key with step 1 into a new file, and put both PEM blocks, old +then new, in the one parameter. ftwcloudd loads both and refuses two keys +under one id; then make the new one again. Ship a Core release that pins +the new configuration. Once boxes have moved to it, drop the old block. + +## Access + +The host role needs nothing new, as for the Noise key: see +[apns.md](apns.md), "Access". diff --git a/deploy/ohttp.sh b/deploy/ohttp.sh new file mode 100755 index 0000000..abca56d --- /dev/null +++ b/deploy/ohttp.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Give ftwcloudd the Oblivious HTTP gateway key. Run as root on the FTW Cloud host +# through SSM, once the key is in Parameter Store +# (deploy/ohttp.md). Safe to re-run. +# +# The key rests only in Parameter Store, as a SecureString under the +# account's aws/ssm key. At every boot, before Docker starts, a unit reads it +# into tmpfs, where compose.yml mounts it into the container. It is never +# written to the volume or the image, and nothing prints it. The host role +# reads it with what it already has, as for the APNs key (deploy/apns.md). +set -euo pipefail + +cat > /usr/local/sbin/ftw-cloud-ohttp <<'FETCH' +#!/bin/bash +# Read the OHTTP gateway key from Parameter Store into tmpfs for ftwcloudd. +# Without the parameter ftwcloudd runs and /v1/ohttp stays off. +set -euo pipefail +export AWS_PAGER="" +umask 077 +dir=/run/ftw-cloud/ohttp +install -d -m 0700 /run/ftw-cloud +new=$(mktemp -d /run/ftw-cloud/ohttp.XXXXXX) +trap 'rm -rf "$new"' EXIT +# At boot the network and the instance's credentials may lag a little. +for try in $(seq 30); do + if err=$(aws ssm get-parameter --region eu-north-1 --name /ftw-cloud/ohttp-key --with-decryption \ + --query Parameter.Value --output text 2>&1 >"$new/ohttp.key"); then + break + fi + case "$err" in *ParameterNotFound*) + echo "no OHTTP key in Parameter Store; /v1/ohttp stays off" + exit 0 ;; + esac + [ "$try" -lt 30 ] || { echo "$err" >&2; exit 1; } + sleep 2 +done +grep -q '^-----BEGIN PRIVATE KEY-----$' "$new/ohttp.key" +# ftw-cloud-own gives it to the user the image runs as, before compose starts. +chown -R 65532:65532 "$new" +chmod 0400 "$new/ohttp.key" +chmod 0500 "$new" +rm -rf "$dir" +mv "$new" "$dir" +trap - EXIT +echo "OHTTP key in place" +FETCH +chmod 0755 /usr/local/sbin/ftw-cloud-ohttp + +cat > /etc/systemd/system/ftw-cloud-ohttp.service <<'UNIT' +[Unit] +Description=FTW Cloud OHTTP gateway key into tmpfs +Before=docker.service +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/ftw-cloud-ohttp + +[Install] +WantedBy=docker.service +UNIT + +systemctl daemon-reload +systemctl enable ftw-cloud-ohttp.service +systemctl restart ftw-cloud-ohttp.service diff --git a/go.mod b/go.mod index 79eb397..766e3fb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261023165843-21e3d16946f1 + github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 35d7169..bf84375 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/srcfl/ftw/go/protocol v0.0.0-20261023165843-21e3d16946f1 h1:OCneVMFORT+ejZeiWAQ+66tHIN0IMXX7MtLtNbF6m1U= github.com/srcfl/ftw/go/protocol v0.0.0-20261023165843-21e3d16946f1/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242 h1:ePRqHUvSp+2YQX99v9+uDO956Cj1CvYxGMbcQnXuUY4= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/energy/energy.go b/internal/energy/energy.go new file mode 100644 index 0000000..99240bf --- /dev/null +++ b/internal/energy/energy.go @@ -0,0 +1,198 @@ +// Package energy is the Oblivious HTTP gateway (RFC 9458) for the anonymous +// daily energy report, and the public view of its totals. +// +// A box seals one report a day to the gateway key and posts it to an +// independent relay, which forwards it here. The gateway sees the report and +// the relay's address, never the box's. It adds the six sums to that date's +// totals and keeps nothing else: no row, logged body or address per report. +// +// A report cannot prove that a real box sent it. Bounds per field and a cap +// per date keep junk to noise. Privacy Pass-style tokens would be the fix. +package energy + +import ( + "encoding/json" + "io" + "mime" + "net/http" + "sync" + "time" + + "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +const ( + // maxRequest is a sealed request: header, key, padded message and tag, + // with room to spare. + maxRequest = 1 << 10 + // Minimum is the group size for a public date, as /v1/fleet/public's. + Minimum = 5 + // rate and burst are one shared budget for every request. All of them + // arrive from the relay, so there is no address worth keeping apart. + rate = 5.0 + burst = 120.0 +) + +type Service struct { + reg *registry.Registry + // gw is nil on a host without a gateway key; the gateway routes then say + // so and the public view still answers. + gw *ohttp.Gateway + keys []byte + now func() time.Time + + mu sync.Mutex + at time.Time + tokens float64 +} + +func New(reg *registry.Registry, gw *ohttp.Gateway) (*Service, error) { + if err := reg.EnergyInit(); err != nil { + return nil, err + } + if err := reg.EnergyPrune(time.Now()); err != nil { + return nil, err + } + s := &Service{reg: reg, gw: gw, now: time.Now, tokens: burst} + if gw != nil { + s.keys = ohttp.MarshalKeys(gw.Configs()...) + } + return s, nil +} + +// ServeHTTP is the gateway, POST /v1/ohttp. A request it cannot open gets a +// plain 400. One it opens gets a sealed answer: 204 when the report is +// added, 400 when it is not a valid report, 429 when its date is full. +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if s.gw == nil { + http.Error(w, "not available yet", http.StatusNotImplemented) + return + } + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); media != ohttp.RequestType { + w.WriteHeader(http.StatusUnsupportedMediaType) + return + } + if !s.allow() { + w.WriteHeader(http.StatusTooManyRequests) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxRequest)) + if err != nil { + w.WriteHeader(http.StatusRequestEntityTooLarge) + return + } + msg, sealer, err := s.gw.Decapsulate(body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + answer, err := sealer.Seal(ohttp.Response{Status: s.add(msg)}.Marshal(energywire.ResponseSize)) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", ohttp.ResponseType) + _, _ = w.Write(answer) +} + +// add reads the one request the gateway answers and adds its report. +func (s *Service) add(msg []byte) int { + req, err := ohttp.ParseRequest(msg) + if err != nil || len(msg) != energywire.RequestSize || req.Method != energywire.Method || req.Scheme != energywire.Scheme || + req.Authority != energywire.Authority || req.Path != energywire.Path || len(req.Fields) != 0 { + return http.StatusBadRequest + } + report, err := energywire.Decode(req.Content, s.now()) + if err != nil { + return http.StatusBadRequest + } + switch ok, err := s.reg.EnergyAdd(report); { + case err != nil: + return http.StatusServiceUnavailable + case !ok: + return http.StatusTooManyRequests + } + return http.StatusNoContent +} + +// Keys serves the gateway's key configuration at +// /.well-known/ohttp-gateway (RFC 9540). Boxes do not read it: a release pins +// the key. It is here for the relay's operator and for anyone checking the +// pin. +func (s *Service) Keys(w http.ResponseWriter, r *http.Request) { + if s.gw == nil { + http.Error(w, "not available yet", http.StatusNotImplemented) + return + } + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("Cache-Control", "public, max-age=86400") + w.Header().Set("Content-Type", ohttp.KeysType) + _, _ = w.Write(s.keys) +} + +// Public is GET /v1/energy/public: the closed dates of the last EnergyDays, +// newest first. A date shows only once no report may name it any more, so +// two readings never differ by one report. A date with fewer than Minimum +// reports shows null. +func (s *Service) Public(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "public, max-age=60") + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + now := s.now().UTC() + today := now.Truncate(24 * time.Hour) + first, last := today.AddDate(0, 0, -registry.EnergyDays), today.AddDate(0, 0, -energywire.AcceptDays-1) + totals, err := s.reg.EnergyTotals(first.Format(time.DateOnly), last.Format(time.DateOnly)) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + days := []map[string]any{} + for d := last; !d.Before(first); d = d.AddDate(0, 0, -1) { + date := d.Format(time.DateOnly) + day, row := totals[date], map[string]any{"date": date, "reports": nil} + for i, name := range energywire.Fields { + row[name] = nil + if day.Reports >= Minimum { + row[name] = day.Sums[i] + } + } + if day.Reports >= Minimum { + row["reports"] = day.Reports + } + days = append(days, row) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"schema": "ftw.energy-public/1", "meaning": "daily reports, not homes", + "observed_at": now.Format(time.RFC3339), "minimum": Minimum, "days": days}) +} + +func (s *Service) Prune(now time.Time) error { return s.reg.EnergyPrune(now) } + +func (s *Service) allow() bool { + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + if !s.at.IsZero() { + s.tokens = min(burst, s.tokens+max(0, now.Sub(s.at).Seconds())*rate) + } + s.at = now + if s.tokens < 1 { + return false + } + s.tokens-- + return true +} diff --git a/internal/energy/energy_test.go b/internal/energy/energy_test.go new file mode 100644 index 0000000..2c3fa7e --- /dev/null +++ b/internal/energy/energy_test.go @@ -0,0 +1,249 @@ +package energy + +import ( + "bytes" + "crypto/ecdh" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +var now = time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC) + +const good = `{"schema":"ftw.energy/1","date":"2026-10-23","pv_generation_kwh":12,"grid_import_kwh":30,"grid_export_kwh":4,"battery_charge_kwh":9,"battery_discharge_kwh":8,"vehicle_charge_kwh":0}` + +func newService(t *testing.T) (*Service, *registry.Registry, ohttp.KeyConfig) { + t.Helper() + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = reg.Close() }) + k, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + gw, err := ohttp.NewGateway(ohttp.Key{ID: 9, Private: k}) + if err != nil { + t.Fatal(err) + } + s, err := New(reg, gw) + if err != nil { + t.Fatal(err) + } + s.now = func() time.Time { return now } + return s, reg, gw.Configs()[0] +} + +func inner(content string) ohttp.Request { + return ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, Path: energywire.Path, Content: []byte(content)} +} + +func post(s *Service, body []byte, contentType string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/v1/ohttp", bytes.NewReader(body)) + r.Header.Set("Content-Type", contentType) + r.RemoteAddr = "198.51.100.8:443" + s.ServeHTTP(w, r) + return w +} + +// send seals msg as a box would and returns the gateway's inner status. +func send(t *testing.T, s *Service, cfg ohttp.KeyConfig, msg []byte) int { + t.Helper() + sealed, reader, err := ohttp.Encapsulate(cfg, msg) + if err != nil { + t.Fatal(err) + } + w := post(s, sealed, ohttp.RequestType) + if w.Code != http.StatusOK || w.Header().Get("Content-Type") != ohttp.ResponseType || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("outer %d %v", w.Code, w.Header()) + } + plain, err := reader.Open(w.Body.Bytes()) + if err != nil { + t.Fatal(err) + } + resp, err := ohttp.ParseResponse(plain) + if err != nil || len(plain) != energywire.ResponseSize { + t.Fatalf("inner %x %v", plain, err) + } + return resp.Status +} + +func TestAReportAddsToItsDate(t *testing.T) { + s, reg, cfg := newService(t) + for range 2 { + if got := send(t, s, cfg, inner(good).Marshal(energywire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("status %d", got) + } + } + totals, err := reg.EnergyTotals("2026-10-23", "2026-10-23") + if d := totals["2026-10-23"]; err != nil || d.Reports != 2 || d.Sums != [6]int64{24, 60, 8, 18, 16, 0} { + t.Fatalf("%+v %v", d, err) + } +} + +// Anything but the one padded request with one valid report gets a sealed +// 400 and adds nothing. +func TestJunkIsRefusedAndAddsNothing(t *testing.T) { + s, reg, cfg := newService(t) + pad := energywire.RequestSize + other := func(f func(*ohttp.Request)) []byte { + r := inner(good) + f(&r) + return r.Marshal(pad) + } + for name, msg := range map[string][]byte{ + "another path": other(func(r *ohttp.Request) { r.Path = "/v1/fleet" }), + "another method": other(func(r *ohttp.Request) { r.Method = "PUT" }), + "another host": other(func(r *ohttp.Request) { r.Authority = "example.com" }), + "plain http": other(func(r *ohttp.Request) { r.Scheme = "http" }), + "a header": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "x-home", Value: "1"}} }), + "not padded": inner(good).Marshal(0), + "padded more": inner(good).Marshal(pad + 1), + "extra field": inner(strings.TrimSuffix(good, "}") + `,"home":"a"}`).Marshal(pad), + "too much solar": inner(strings.Replace(good, `"pv_generation_kwh":12`, `"pv_generation_kwh":1001`, 1)).Marshal(pad), + "negative": inner(strings.Replace(good, `:12,`, `:-12,`, 1)).Marshal(pad), + "today": inner(strings.Replace(good, "2026-10-23", "2026-10-24", 1)).Marshal(pad), + "four days back": inner(strings.Replace(good, "2026-10-23", "2026-10-20", 1)).Marshal(pad), + "other schema": inner(strings.Replace(good, "ftw.energy/1", "ftw.energy/2", 1)).Marshal(pad), + "a response": ohttp.Response{Status: 200, Content: []byte(good)}.Marshal(pad), + "empty": nil, + } { + if got := send(t, s, cfg, msg); got != http.StatusBadRequest { + t.Fatalf("%s: %d", name, got) + } + } + sealed, _, _ := ohttp.Encapsulate(cfg, inner(good).Marshal(pad)) + tampered := bytes.Clone(sealed) + tampered[len(tampered)-1] ^= 1 + for name, c := range map[string]struct { + body []byte + ctype string + code int + }{ + "tampered": {tampered, ohttp.RequestType, http.StatusBadRequest}, + "random bytes": {bytes.Repeat([]byte{9}, 400), ohttp.RequestType, http.StatusBadRequest}, + "plain JSON": {[]byte(good), "application/json", http.StatusUnsupportedMediaType}, + "oversized": {make([]byte, maxRequest+1), ohttp.RequestType, http.StatusRequestEntityTooLarge}, + } { + w := post(s, c.body, c.ctype) + if w.Code != c.code || w.Header().Get("Content-Type") == ohttp.ResponseType { + t.Fatalf("%s: %d", name, w.Code) + } + } + w := httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/ohttp", nil)) + if w.Code != http.StatusMethodNotAllowed { + t.Fatalf("GET %d", w.Code) + } + if totals, _ := reg.EnergyTotals("2026-01-01", "2026-12-31"); len(totals) != 0 { + t.Fatalf("junk reached the totals: %v", totals) + } +} + +// One budget for all requests, since they all come from the relay. +func TestRequestsShareOneBudget(t *testing.T) { + s, _, cfg := newService(t) + sealed, _, _ := ohttp.Encapsulate(cfg, inner(good).Marshal(energywire.RequestSize)) + for i := range int(burst) { + if w := post(s, sealed, ohttp.RequestType); w.Code == http.StatusTooManyRequests { + t.Fatalf("limited after %d", i) + } + } + if w := post(s, sealed, ohttp.RequestType); w.Code != http.StatusTooManyRequests { + t.Fatalf("over budget: %d", w.Code) + } + now = now.Add(time.Second) + defer func() { now = now.Add(-time.Second) }() + if w := post(s, sealed, ohttp.RequestType); w.Code != http.StatusOK { + t.Fatalf("after a second: %d", w.Code) + } +} + +type publicView struct { + Schema string `json:"schema"` + Meaning string `json:"meaning"` + ObservedAt string `json:"observed_at"` + Minimum int `json:"minimum"` + Days []map[string]any `json:"days"` +} + +// A date is public once it is closed for reports and holds at least five. +func TestAPublicDateIsClosedAndHoldsFiveReports(t *testing.T) { + s, reg, _ := newService(t) + add := func(date string, n int) { + var r energywire.Report + _ = json.Unmarshal([]byte(strings.Replace(good, "2026-10-23", date, 1)), &r) + for range n { + if ok, err := reg.EnergyAdd(r); err != nil || !ok { + t.Fatal(err) + } + } + } + add("2026-10-18", 5) + add("2026-10-19", 4) + add("2026-10-23", 9) // still open for reports + add("2026-09-23", 9) // past the window + + w := httptest.NewRecorder() + s.Public(w, httptest.NewRequest(http.MethodGet, "/v1/energy/public", nil)) + var v publicView + if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil { + t.Fatal(err) + } + if v.Schema != "ftw.energy-public/1" || v.Meaning != "daily reports, not homes" || v.Minimum != 5 || v.ObservedAt != "2026-10-24T09:00:00Z" { + t.Fatalf("view %+v", v) + } + if len(v.Days) != 27 || v.Days[0]["date"] != "2026-10-20" || v.Days[26]["date"] != "2026-09-24" { + t.Fatalf("%d days from %v to %v", len(v.Days), v.Days[0]["date"], v.Days[len(v.Days)-1]["date"]) + } + byDate := map[string]map[string]any{} + for _, d := range v.Days { + if len(d) != 2+len(energywire.Fields) { + t.Fatalf("fields %v", d) + } + byDate[d["date"].(string)] = d + } + if d := byDate["2026-10-18"]; d["reports"] != 5.0 || d["pv_generation_kwh"] != 60.0 || d["vehicle_charge_kwh"] != 0.0 { + t.Fatalf("five reports: %v", d) + } + for _, date := range []string{"2026-10-19", "2026-10-20"} { + for k, v := range byDate[date] { + if k != "date" && v != nil { + t.Fatalf("%s shows %s=%v", date, k, v) + } + } + } + if _, open := byDate["2026-10-23"]; open { + t.Fatal("a date still open for reports is public") + } +} + +func TestTheKeyConfigurationIsServed(t *testing.T) { + s, _, cfg := newService(t) + w := httptest.NewRecorder() + s.Keys(w, httptest.NewRequest(http.MethodGet, "/.well-known/ohttp-gateway", nil)) + if w.Header().Get("Content-Type") != ohttp.KeysType || !bytes.Equal(w.Body.Bytes(), ohttp.MarshalKeys(cfg)) { + t.Fatalf("%v %x", w.Header(), w.Body.Bytes()) + } + off, err := New(s.reg, nil) + if err != nil { + t.Fatal(err) + } + for _, h := range []http.HandlerFunc{off.ServeHTTP, off.Keys} { + w := httptest.NewRecorder() + h(w, httptest.NewRequest(http.MethodPost, "/v1/ohttp", nil)) + if w.Code != http.StatusNotImplemented { + t.Fatalf("without a key: %d", w.Code) + } + } +} diff --git a/internal/registry/energy.go b/internal/registry/energy.go new file mode 100644 index 0000000..af48fd8 --- /dev/null +++ b/internal/registry/energy.go @@ -0,0 +1,79 @@ +package registry + +import ( + "time" + + "github.com/srcfl/ftw/go/protocol/energywire" +) + +// EnergyDays is how long a date's energy totals are kept. +const EnergyDays = 30 + +// EnergyInit creates only daily totals: one row per date, a count and six +// sums. No report, address or time of arrival goes on disk. +func (r *Registry) EnergyInit() error { + _, err := r.db.Exec(`CREATE TABLE IF NOT EXISTS energy_days ( + date TEXT PRIMARY KEY, + reports INTEGER NOT NULL, + pv_generation_kwh INTEGER NOT NULL, + grid_import_kwh INTEGER NOT NULL, + grid_export_kwh INTEGER NOT NULL, + battery_charge_kwh INTEGER NOT NULL, + battery_discharge_kwh INTEGER NOT NULL, + vehicle_charge_kwh INTEGER NOT NULL)`) + return err +} + +// EnergyAdd adds one report to its date's totals in one statement. False +// means the date already holds energywire.MaxReports. +func (r *Registry) EnergyAdd(rep energywire.Report) (bool, error) { + s := rep.Sums() + res, err := r.db.Exec(`INSERT INTO energy_days VALUES (?, 1, ?, ?, ?, ?, ?, ?) + ON CONFLICT(date) DO UPDATE SET reports = reports + 1, + pv_generation_kwh = pv_generation_kwh + excluded.pv_generation_kwh, + grid_import_kwh = grid_import_kwh + excluded.grid_import_kwh, + grid_export_kwh = grid_export_kwh + excluded.grid_export_kwh, + battery_charge_kwh = battery_charge_kwh + excluded.battery_charge_kwh, + battery_discharge_kwh = battery_discharge_kwh + excluded.battery_discharge_kwh, + vehicle_charge_kwh = vehicle_charge_kwh + excluded.vehicle_charge_kwh + WHERE reports < ?`, + rep.Date, s[0], s[1], s[2], s[3], s[4], s[5], energywire.MaxReports) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n == 1, err +} + +// EnergyDay is one date's totals, sums in energywire.Fields order. +type EnergyDay struct { + Date string + Reports int64 + Sums [6]int64 +} + +// EnergyTotals reads the dates from first to last, both included. +func (r *Registry) EnergyTotals(first, last string) (map[string]EnergyDay, error) { + rows, err := r.rd.Query(`SELECT date, reports, pv_generation_kwh, grid_import_kwh, grid_export_kwh, + battery_charge_kwh, battery_discharge_kwh, vehicle_charge_kwh + FROM energy_days WHERE date >= ? AND date <= ? LIMIT ?`, first, last, EnergyDays+1) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]EnergyDay{} + for rows.Next() { + var d EnergyDay + if err := rows.Scan(&d.Date, &d.Reports, &d.Sums[0], &d.Sums[1], &d.Sums[2], &d.Sums[3], &d.Sums[4], &d.Sums[5]); err != nil { + return nil, err + } + out[d.Date] = d + } + return out, rows.Err() +} + +// EnergyPrune drops dates older than EnergyDays, even when no report comes. +func (r *Registry) EnergyPrune(now time.Time) error { + _, err := r.db.Exec(`DELETE FROM energy_days WHERE date < ?`, now.UTC().AddDate(0, 0, -EnergyDays).Format(time.DateOnly)) + return err +} diff --git a/internal/registry/energy_test.go b/internal/registry/energy_test.go new file mode 100644 index 0000000..61324ac --- /dev/null +++ b/internal/registry/energy_test.go @@ -0,0 +1,57 @@ +package registry + +import ( + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol/energywire" +) + +// Reports leave one row per date, a count and six sums, and nothing that +// was in any one of them. +func TestEnergyKeepsOnlyDailyTotals(t *testing.T) { + r, err := Open(filepath.Join(t.TempDir(), "registry.db")) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if err := r.EnergyInit(); err != nil { + t.Fatal(err) + } + rep := energywire.Report{Schema: energywire.Schema, Date: "2026-10-23", PVGeneration: 7, GridImport: 1} + for i := range 10 { + rep.Date = []string{"2026-10-22", "2026-10-23"}[i%2] + if ok, err := r.EnergyAdd(rep); err != nil || !ok { + t.Fatal(err) + } + } + var rows, columns int + if err := r.db.QueryRow(`SELECT COUNT(*) FROM energy_days`).Scan(&rows); err != nil || rows != 2 { + t.Fatalf("rows %d %v", rows, err) + } + if err := r.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('energy_days')`).Scan(&columns); err != nil || columns != 2+len(energywire.Fields) { + t.Fatalf("columns %d %v", columns, err) + } + totals, err := r.EnergyTotals("2026-10-22", "2026-10-23") + if d := totals["2026-10-23"]; err != nil || d.Reports != 5 || d.Sums[0] != 35 || d.Sums[1] != 5 { + t.Fatalf("%+v %v", d, err) + } + + // A full date takes no more. + if _, err := r.db.Exec(`UPDATE energy_days SET reports = ? WHERE date = '2026-10-23'`, energywire.MaxReports); err != nil { + t.Fatal(err) + } + if ok, err := r.EnergyAdd(rep); err != nil || ok { + t.Fatalf("a full date took a report: %v", err) + } + + // Dates expire by age, even when no report arrives. + if err := r.EnergyPrune(time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + totals, _ = r.EnergyTotals("2026-10-01", "2026-10-31") + if _, ok := totals["2026-10-22"]; ok || len(totals) != 1 { + t.Fatalf("after pruning %v", totals) + } +} diff --git a/internal/server/energy_e2e_test.go b/internal/server/energy_e2e_test.go new file mode 100644 index 0000000..5a45a3f --- /dev/null +++ b/internal/server/energy_e2e_test.go @@ -0,0 +1,103 @@ +package server + +import ( + "bytes" + "crypto/ecdh" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/ohttp" +) + +func testGateway(t *testing.T) *ohttp.Gateway { + t.Helper() + k, err := ecdh.X25519().GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + g, err := ohttp.NewGateway(ohttp.Key{ID: 3, Private: k}) + if err != nil { + t.Fatal(err) + } + return g +} + +// Box to relay to gateway to totals. The box half is what Core sends: the +// key configuration from /.well-known/ohttp-gateway, the one inner request, +// padded. The relay forwards only the body and its type, and never sees the +// report. ftwcloudd adds it to the date's totals and answers a sealed 204. +func TestAnEnergyReportReachesTheTotalsThroughARelay(t *testing.T) { + s, _ := newServer(t) + cloud := httptest.NewServer(s.Handler()) + defer cloud.Close() + + var seen [][]byte + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + seen = append(seen, body) + resp, err := http.Post(cloud.URL+"/v1/ohttp", r.Header.Get("Content-Type"), bytes.NewReader(body)) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + defer resp.Body.Close() + w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + defer relay.Close() + + resp, err := http.Get(cloud.URL + "/.well-known/ohttp-gateway") + if err != nil { + t.Fatal(err) + } + keys, _ := io.ReadAll(resp.Body) + resp.Body.Close() + configs, err := ohttp.ParseKeys(keys) + if err != nil || len(configs) != 1 || resp.Header.Get("Content-Type") != ohttp.KeysType { + t.Fatalf("key configuration %x %v", keys, err) + } + + yesterday := time.Now().UTC().AddDate(0, 0, -1).Format(time.DateOnly) + report := energywire.Report{Schema: energywire.Schema, Date: yesterday, PVGeneration: 12, GridImport: 30, GridExport: 4, BatteryCharge: 9, BatteryDischarge: 8} + for i := range 3 { + body, _ := json.Marshal(report) + inner := ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, Path: energywire.Path, Content: body}.Marshal(energywire.RequestSize) + sealed, reader, err := ohttp.Encapsulate(configs[0], inner) + if err != nil { + t.Fatal(err) + } + resp, err := http.Post(relay.URL, ohttp.RequestType, bytes.NewReader(sealed)) + if err != nil { + t.Fatal(err) + } + answer, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != ohttp.ResponseType { + t.Fatalf("relay got %d %v", resp.StatusCode, resp.Header) + } + msg, err := reader.Open(answer) + if err != nil { + t.Fatal(err) + } + if got, err := ohttp.ParseResponse(msg); err != nil || got.Status != http.StatusNoContent || len(msg) != energywire.ResponseSize { + t.Fatalf("report %d: %+v %v", i, got, err) + } + if bytes.Contains(seen[i], []byte("kwh")) || bytes.Contains(seen[i], []byte(yesterday)) { + t.Fatal("the relay saw the report") + } + } + totals, err := s.Registry.EnergyTotals(yesterday, yesterday) + day := totals[yesterday] + if err != nil || day.Reports != 3 || day.Sums != [6]int64{36, 90, 12, 27, 24, 0} { + t.Fatalf("totals %+v %v", day, err) + } + if len(seen[0]) != len(seen[1]) { + t.Fatal("two reports differ in length on the relay") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 7b5fcb8..3c5ca5e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -5,6 +5,9 @@ // /v1/app a phone's Noise session to the service (package appsession) // /v1/app/once a watch's one-shot read, Noise inside one POST (package appsession) // /v1/now a widget's token read (package token) +// /v1/ohttp the Oblivious HTTP gateway for anonymous daily energy reports (package energy) +// /.well-known/ohttp-gateway that gateway's key configuration +// /v1/fleet, /v1/fleet/public, /v1/energy/public anonymous daily totals // /source the source of the running service, as AGPL-3.0 asks // /healthz liveness, and nothing about any home // /oauth/*, /.well-known/oauth-* the authorization server for assistants (package oauth) @@ -26,6 +29,7 @@ import ( "time" "github.com/srcfl/ftw-cloud/internal/appsession" + "github.com/srcfl/ftw-cloud/internal/energy" "github.com/srcfl/ftw-cloud/internal/oauth" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/store" @@ -37,7 +41,8 @@ import ( const SourceArchive = "/usr/share/ftw-cloud/source.tar.gz" // Routes are every path the service answers. -var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/fleet/public", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) +var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/fleet/public", + "/v1/ohttp", "/.well-known/ohttp-gateway", "/v1/energy/public", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) // devRoutes adds a local inspection route in binaries built with // -tags ftwclouddev. It is nil in every other build. @@ -58,6 +63,7 @@ type Server struct { // they call. Nil serves none. OAuth *oauth.Server Fleet http.Handler + Energy *energy.Service Statistics http.Handler MCP http.Handler Log *slog.Logger @@ -100,6 +106,13 @@ func (s *Server) Handler() http.Handler { } else { mux.HandleFunc("/v1/fleet", notYet) } + if s.Energy != nil { + mux.Handle("/v1/ohttp", s.Energy) + mux.HandleFunc("/.well-known/ohttp-gateway", s.Energy.Keys) + mux.HandleFunc("/v1/energy/public", s.Energy.Public) + } else { + mux.HandleFunc("/v1/ohttp", notYet) + } if s.Statistics != nil { mux.Handle("/v1/stats", s.Statistics) } else { @@ -197,6 +210,11 @@ func (s *Server) Upkeep(ctx context.Context, every time.Duration) { s.Log.Warn("fleet upkeep failed") } } + if s.Energy != nil { + if err := s.Energy.Prune(time.Now()); err != nil { + s.Log.Warn("energy upkeep failed") + } + } if f, ok := s.Statistics.(interface{ Prune(time.Time) error }); ok { if err := f.Prune(time.Now()); err != nil { s.Log.Warn("statistics upkeep failed") diff --git a/internal/server/server_test.go b/internal/server/server_test.go index c3ec50c..9e1e01d 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/srcfl/ftw-cloud/internal/appsession" + "github.com/srcfl/ftw-cloud/internal/energy" "github.com/srcfl/ftw-cloud/internal/fleet" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/statistics" @@ -47,6 +48,9 @@ func newServer(t *testing.T) (*Server, string) { if err != nil { t.Fatal(err) } + if s.Energy, err = energy.New(reg, testGateway(t)); err != nil { + t.Fatal(err) + } up.Statistics = true s.Statistics.(*statistics.Service).Enabled = true // A home with readings in it, held the way a box's uplink holds them. From 506ee104ddea96ddf2ff0d55240497f4031fd812 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 08:38:06 +0200 Subject: [PATCH 2/8] feat: one gateway route for both anonymous daily reports Fredrik's change of scope, 24 October 2026: the fleet report moves onto the same relay and gateway as the energy report, and the two stay apart. - internal/energy becomes internal/gateway. POST /v1/ohttp reads the one padded inner request (go/protocol/reportwire), picks the decoder by schema, and adds a fleet report to the existing fleet_days totals that /v1/fleet/public reads and an energy report to energy_days. Nothing is kept per report. - POST /v1/fleet stays for boxes on a Core release without a relay; it can go one stable Core release after the first that pins the relay. - The request cap fits the 1,591-byte sealed request; README and deploy/ohttp.md describe both reports and the two-per-day traffic. - go/protocol pinned at srcfl/ftw 4cfd65f7. Co-Authored-By: Claude Opus 5.5 --- README.md | 73 +++++++++++-------- cmd/ftwcloudd/main.go | 16 ++-- cmd/ftwcloudd/ohttp.go | 2 +- deploy/README.md | 2 +- deploy/compose.yml | 2 +- deploy/ohttp.md | 21 +++--- go.mod | 2 +- go.sum | 2 + internal/fleet/fleet.go | 5 ++ .../{energy/energy.go => gateway/gateway.go} | 59 ++++++++++----- .../gateway_test.go} | 73 +++++++++++++------ ...energy_e2e_test.go => reports_e2e_test.go} | 39 +++++++--- internal/server/server.go | 18 ++--- internal/server/server_test.go | 4 +- 14 files changed, 204 insertions(+), 114 deletions(-) rename internal/{energy/energy.go => gateway/gateway.go} (78%) rename internal/{energy/energy_test.go => gateway/gateway_test.go} (69%) rename internal/server/{energy_e2e_test.go => reports_e2e_test.go} (63%) diff --git a/README.md b/README.md index 210cdad..642f51a 100644 --- a/README.md +++ b/README.md @@ -452,9 +452,12 @@ with `go list -deps`, so no control code can reach this build. only (AGPL-3.0-only). See [LICENSE](LICENSE) and [CONTRIBUTING.md](CONTRIBUTING.md). -Anonymous box reports use `POST /v1/fleet`. It accepts only the seven -`ftw.fleet/1` fields and reduces each report into bounded daily counts for -90 days. It stores no raw report, address or home id. An operator can read +Anonymous box reports use `POST /v1/fleet`, or, from a Core release that +names a relay, `POST /v1/ohttp` (see "Anonymous daily reports" below). The +direct route sees the box's address; it stays for older releases and can go +one stable Core release after the first that pins the relay. Either way it +accepts only the seven `ftw.fleet/1` fields and reduces each report into +bounded daily counts for 90 days. It stores no raw report, address or home id. An operator can read `/fleet/stats` only on the loopback metrics listener, with `docker exec ftw-cloud-ftwcloudd-1 ftwcloudd fleet`. These counts measure reports, not unique boxes. @@ -513,28 +516,37 @@ energy activation. Updated Core sends it only after the current project statistics answer; legacy versions retain their existing daily-report setting. Averaging seven complete UTC days gives an estimate of active reporting installations, never people or all installations ever made. -## Anonymous daily energy - -Energy on stats.ftw.energy comes from a separate report that is not personal -data to this service, so it needs no C8 activation. Once per closed UTC day, -1 to 12 hours after it closes, a box sends the date and six whole-kWh sums: -solar generation, grid import and export, battery charge and discharge, and -vehicle charge. Nothing else, and no field the fleet report carries. A day -with a gap in the box's energy ledger is skipped. Core sends it only under -the fleet report's answer and never for a site a partner controls -(`go/internal/energyreport` in srcfl/ftw). - -The box seals the report with Oblivious HTTP (RFC 9458) to the gateway key -its release pins, and posts it to an independent relay -([deploy/ohttp.md](deploy/ohttp.md)). The relay sees the box's address and a -ciphertext of fixed length. `POST /v1/ohttp` (`internal/energy`) sees the -report and the relay's address. It answers a sealed 204, or 400 for anything -but the one padded request with one valid report: exact fields, at most -1,000 kWh each, a date in the last three closed days. It adds the sums to -that date's row and keeps nothing per report: no row, logged body or -address. All requests share one rate budget, and a date takes at most -100,000 reports. `GET /.well-known/ohttp-gateway` serves the key -configuration for the relay operator and for anyone checking the pin. +## Anonymous daily reports + +Statistics from a box are two anonymous daily reports, each sent alone and at +its own random time, never in one message. With FTW Cloud they are part of +Cloud; without it they are on until the owner turns them off on the box. The +rule lives in Core (`cloudlink.SharesStatistics` in srcfl/ftw). + +- The fleet report (`ftw.fleet/1`): version, channel, drivers, battery size, + price zone and install age, into the counts `/v1/fleet/public` publishes. +- The energy report (`ftw.energy/1`): once per closed UTC day, 1 to 12 hours + after it closes, the date and six whole-kWh sums (solar generation, grid + import and export, battery charge and discharge, vehicle charge). A day + with a gap in the box's energy ledger is skipped. + +They never travel together because, with a fleet of a few boxes, a driver set +and a price zone can point at one home, and energy beside them would then be +that home's. + +From a Core release that names a relay, the box seals each report with +Oblivious HTTP (RFC 9458) to the gateway key its release pins, and posts it +to an independent relay ([deploy/ohttp.md](deploy/ohttp.md)). The relay sees +the box's address and a ciphertext of fixed length (1,591 bytes). +`POST /v1/ohttp` (`internal/gateway`) sees the report and the relay's +address. It reads the one padded inner request, chooses the decoder by +schema, adds the report to its own day totals and answers a sealed 204, or +400 for anything else: exact fields, and for energy at most 1,000 kWh each +and a date in the last three closed days. It keeps nothing per report: no +row, logged body or address. All requests share one rate budget, and a day +takes at most 100,000 reports of each kind. +`GET /.well-known/ohttp-gateway` serves the key configuration for the relay +operator and for anyone checking the pin. `GET /v1/energy/public` lists the closed dates of the last 30 days, newest first: the report count and the six sums. A date appears only once no report @@ -543,7 +555,7 @@ fewer than 5 reports shows null, as `/v1/fleet/public` does. Rows go after 30 days. These are reports, not homes. Limits, plainly. A report cannot prove that a real box sent it, and the -relay could replay one it forwarded; the bounds and the cap keep both to +relay could replay one it forwarded; the bounds and the caps keep both to noise. Privacy Pass-style tokens are the later fix and are not built. The relay and Sourceful together could join reports to addresses by time. This service also holds readings from homes with the readable copy on, including @@ -551,7 +563,8 @@ service also holds readings from homes with the readable copy on, including the gateway does not do this and keeps nothing to match with, but that is a rule in code, not one the cryptography enforces. -Open for Fredrik: whether reports that are anonymous to Sourceful still need -the owner's yes under the ePrivacy rule on reading terminal equipment -(Swedish LEK 9:28). The design keeps the existing yes, so it is covered -either way. +Open for Fredrik's lawyer: reports that are anonymous to Sourceful through +OHTTP should be outside GDPR, but reading data from the owner's equipment to +make them may still need consent under the ePrivacy rule (Swedish LEK, +chapter 9, section 28). If it does, neither the opt-out for boxes without +Cloud nor the requirement with Cloud holds. diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 7712a1f..6a0e1ca 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -38,8 +38,8 @@ import ( "github.com/srcfl/ftw-cloud/internal/apns" "github.com/srcfl/ftw-cloud/internal/appsession" - "github.com/srcfl/ftw-cloud/internal/energy" "github.com/srcfl/ftw-cloud/internal/fleet" + "github.com/srcfl/ftw-cloud/internal/gateway" "github.com/srcfl/ftw-cloud/internal/keys" "github.com/srcfl/ftw-cloud/internal/mcp" "github.com/srcfl/ftw-cloud/internal/metrics" @@ -125,11 +125,11 @@ func serve(args []string) error { } else if err != nil { return err } - // The energy reports' gateway takes its keys the same way. Without them - // boxes send nothing that reaches here, and the public totals still answer. - gateway, err := loadOHTTPKeys(*ohttpPath) + // The anonymous reports' gateway takes its keys the same way. Without them + // no report reaches here, and the public totals still answer. + ohttpKeys, err := loadOHTTPKeys(*ohttpPath) if errors.Is(err, os.ErrNotExist) { - log.Warn("no OHTTP key: the energy report gateway is off until one is installed", "path", *ohttpPath) + log.Warn("no OHTTP key: the gateway for the anonymous reports is off until one is installed", "path", *ohttpPath) } else if err != nil { return err } @@ -175,13 +175,13 @@ func serve(args []string) error { if err != nil { return err } - energySvc, err := energy.New(reg, gateway) + gatewaySvc, err := gateway.New(reg, ohttpKeys) if err != nil { return err } tokens := token.New(st, reg, log) tokens.Shards = shards - srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Energy: energySvc, Statistics: usageSvc, Log: log} + srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Gateway: gatewaySvc, Statistics: usageSvc, Log: log} // Assistants: the authorization server, with agent keys in ftwkeysd. agentKeys := keys.Dial(*keysSocket) @@ -402,7 +402,7 @@ func serve(args []string) error { log.Info("ftwcloudd listening", "addr", *listen, "h3", srv.AltSvc != "", "quic_sessions", door != nil && door.Box != nil, "shard", shards.Self(), "shards", shards.Shards(), "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public()), - "ohttp_keys", ohttpPins(gateway)) + "ohttp_keys", ohttpPins(ohttpKeys)) err = httpSrv.ListenAndServeTLS("", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { return err diff --git a/cmd/ftwcloudd/ohttp.go b/cmd/ftwcloudd/ohttp.go index 7bacd36..0489bb9 100644 --- a/cmd/ftwcloudd/ohttp.go +++ b/cmd/ftwcloudd/ohttp.go @@ -75,7 +75,7 @@ func ohttpPins(gw *ohttp.Gateway) string { } // keygenOHTTP writes a new gateway key, never over an existing file, and -// prints its configuration: the string energyreport.GatewayKey in srcfl/ftw +// prints its configuration: the string statsrelay.GatewayKey in srcfl/ftw // pins. For a rotation, put the new block after the old one in the same // parameter; ftwcloudd refuses two keys under one id, and then a new key is // needed. diff --git a/deploy/README.md b/deploy/README.md index 14d6655..b36ad15 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -382,7 +382,7 @@ Cloud data; the images on `ghcr.io` hold only code. ## Not here yet The APNs key ([apns.md](apns.md)), the Noise key phones dial -([noise.md](noise.md)) and the energy reports' gateway key +([noise.md](noise.md)) and the anonymous reports' gateway key ([ohttp.md](ohttp.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the security group ([http3.md](http3.md)). Widget tokens need no secret: the registry holds their hashes. Statistics (C8) have their own diff --git a/deploy/compose.yml b/deploy/compose.yml index 421d63b..cf23143 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -51,7 +51,7 @@ services: # boot by deploy/noise.sh. Empty means /v1/app is off, and the # service runs. - /run/ftw-cloud/noise:/run/secrets/noise:ro - # The energy reports' Oblivious HTTP gateway key, read from Parameter + # The anonymous reports' Oblivious HTTP gateway key, read from Parameter # Store into tmpfs at boot by deploy/ohttp.sh. Empty means /v1/ohttp # is off, and the service runs. - /run/ftw-cloud/ohttp:/run/secrets/ohttp:ro diff --git a/deploy/ohttp.md b/deploy/ohttp.md index 7874666..d978edd 100644 --- a/deploy/ohttp.md +++ b/deploy/ohttp.md @@ -1,8 +1,9 @@ -# The energy reports' gateway key and relay +# The anonymous reports' gateway key and relay -A box sends one anonymous energy report a day, sealed with Oblivious HTTP -(RFC 9458) to this service's gateway key and posted to a relay that someone -other than Sourceful runs. The relay forwards it to `POST /v1/ohttp`. The +A box sends two anonymous reports a day, a fleet report and an energy report, +each alone and sealed with Oblivious HTTP (RFC 9458) to this service's +gateway key, and posts each to a relay that someone other than Sourceful +runs. The relay forwards it to `POST /v1/ohttp`. The relay sees the box's address but not the report; this service sees the report but not the address. Without the key, `/v1/ohttp` answers 501 and the public totals still answer. @@ -74,9 +75,9 @@ Send them: - **Key configuration:** `https://cloud.ftw.energy/.well-known/ohttp-gateway` (`application/ohttp-keys`), or the string from step 1. The suite is DHKEM(X25519, HKDF-SHA256), HKDF-SHA256, AES-128-GCM. -- **Traffic:** `POST` with `message/ohttp-req`, always 375 bytes; answers - are `message/ohttp-res`, always 48 bytes. One request per box per day, - spread over the 12 hours after 00:00 UTC. +- **Traffic:** `POST` with `message/ohttp-req`, always 1,591 bytes; answers + are `message/ohttp-res`, always 48 bytes. Two requests per box per day, at + separate random times. Ask them to confirm, in writing: @@ -92,9 +93,11 @@ They hand back the **relay URL** the boxes post to. Cloudflare's form is ## 5. Pin both in Core -In srcfl/ftw, `go/internal/energyreport/energyreport.go`: `RelayURL` takes the +In srcfl/ftw, `go/internal/statsrelay/statsrelay.go`: `RelayURL` takes the relay URL from step 4 and `GatewayKey` the string from step 1. Boxes on the -release that carries both start sending; older releases send nothing. +release that carries both send both reports through the relay; older +releases send no energy report and post the fleet report straight to +`/v1/fleet`. ## Rotating diff --git a/go.mod b/go.mod index 766e3fb..0a43b22 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242 + github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index bf84375..8918c5c 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/srcfl/ftw/go/protocol v0.0.0-20261023165843-21e3d16946f1 h1:OCneVMFOR github.com/srcfl/ftw/go/protocol v0.0.0-20261023165843-21e3d16946f1/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242 h1:ePRqHUvSp+2YQX99v9+uDO956Cj1CvYxGMbcQnXuUY4= github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95 h1:VMpQKeK7HImzVWeTNIuo2cDl45WFWhFttR/X+hYDOY0= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/fleet/fleet.go b/internal/fleet/fleet.go index d5a1102..1c34c88 100644 --- a/internal/fleet/fleet.go +++ b/internal/fleet/fleet.go @@ -31,6 +31,11 @@ func New(reg *registry.Registry) (*Service, error) { } // ServeHTTP accepts reports only. Public requests never read totals. +// +// This is the direct path, which sees the box's address: for boxes on a Core +// release that names no relay. A release that pins one sends the fleet report +// through /v1/ohttp (package gateway) into the same totals. Remove this route +// one stable Core release after the first that pins the relay. func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") if r.Method != http.MethodPost { diff --git a/internal/energy/energy.go b/internal/gateway/gateway.go similarity index 78% rename from internal/energy/energy.go rename to internal/gateway/gateway.go index 99240bf..781b516 100644 --- a/internal/energy/energy.go +++ b/internal/gateway/gateway.go @@ -1,14 +1,16 @@ -// Package energy is the Oblivious HTTP gateway (RFC 9458) for the anonymous -// daily energy report, and the public view of its totals. +// Package gateway is the Oblivious HTTP gateway (RFC 9458) for the box's two +// anonymous daily reports, and the public view of the energy totals. // -// A box seals one report a day to the gateway key and posts it to an +// A box seals each report alone to the gateway key and posts it to an // independent relay, which forwards it here. The gateway sees the report and -// the relay's address, never the box's. It adds the six sums to that date's -// totals and keeps nothing else: no row, logged body or address per report. +// the relay's address, never the box's. It reads the report's schema, adds a +// fleet report to the fleet's day totals and an energy report to the +// energy's, and keeps nothing else: no row, logged body or address per +// report. // // A report cannot prove that a real box sent it. Bounds per field and a cap // per date keep junk to noise. Privacy Pass-style tokens would be the fix. -package energy +package gateway import ( "encoding/json" @@ -20,13 +22,15 @@ import ( "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" ) const ( - // maxRequest is a sealed request: header, key, padded message and tag, - // with room to spare. - maxRequest = 1 << 10 + // maxRequest is a sealed request, 1,591 bytes (header, key, padded + // message and tag), with room to spare. + maxRequest = 2 << 10 // Minimum is the group size for a public date, as /v1/fleet/public's. Minimum = 5 // rate and burst are one shared budget for every request. All of them @@ -49,6 +53,9 @@ type Service struct { } func New(reg *registry.Registry, gw *ohttp.Gateway) (*Service, error) { + if err := reg.FleetInit(); err != nil { + return nil, err + } if err := reg.EnergyInit(); err != nil { return nil, err } @@ -64,7 +71,7 @@ func New(reg *registry.Registry, gw *ohttp.Gateway) (*Service, error) { // ServeHTTP is the gateway, POST /v1/ohttp. A request it cannot open gets a // plain 400. One it opens gets a sealed answer: 204 when the report is -// added, 400 when it is not a valid report, 429 when its date is full. +// added, 400 when it is not one valid report, 429 when its day is full. func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") if s.gw == nil { @@ -94,7 +101,7 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - answer, err := sealer.Seal(ohttp.Response{Status: s.add(msg)}.Marshal(energywire.ResponseSize)) + answer, err := sealer.Seal(reportwire.Response(s.add(msg))) if err != nil { w.WriteHeader(http.StatusInternalServerError) return @@ -103,21 +110,35 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(answer) } -// add reads the one request the gateway answers and adds its report. +// add reads the one request the gateway answers and adds its report to the +// day totals of its kind. func (s *Service) add(msg []byte) int { - req, err := ohttp.ParseRequest(msg) - if err != nil || len(msg) != energywire.RequestSize || req.Method != energywire.Method || req.Scheme != energywire.Scheme || - req.Authority != energywire.Authority || req.Path != energywire.Path || len(req.Fields) != 0 { + body, ok := reportwire.Body(msg) + if !ok { return http.StatusBadRequest } - report, err := energywire.Decode(req.Content, s.now()) - if err != nil { + var added bool + var err error + switch reportwire.Schema(body) { + case fleetwire.Schema: + report, decodeErr := fleetwire.Decode(body) + if decodeErr != nil { + return http.StatusBadRequest + } + added, err = s.reg.FleetPut(report, s.now()) + case energywire.Schema: + report, decodeErr := energywire.Decode(body, s.now()) + if decodeErr != nil { + return http.StatusBadRequest + } + added, err = s.reg.EnergyAdd(report) + default: return http.StatusBadRequest } - switch ok, err := s.reg.EnergyAdd(report); { + switch { case err != nil: return http.StatusServiceUnavailable - case !ok: + case !added: return http.StatusTooManyRequests } return http.StatusNoContent diff --git a/internal/energy/energy_test.go b/internal/gateway/gateway_test.go similarity index 69% rename from internal/energy/energy_test.go rename to internal/gateway/gateway_test.go index 2c3fa7e..7c973c0 100644 --- a/internal/energy/energy_test.go +++ b/internal/gateway/gateway_test.go @@ -1,4 +1,4 @@ -package energy +package gateway import ( "bytes" @@ -13,7 +13,9 @@ import ( "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" ) var now = time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC) @@ -44,9 +46,11 @@ func newService(t *testing.T) (*Service, *registry.Registry, ohttp.KeyConfig) { } func inner(content string) ohttp.Request { - return ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, Path: energywire.Path, Content: []byte(content)} + return ohttp.Request{Method: reportwire.Method, Scheme: reportwire.Scheme, Authority: reportwire.Authority, Path: reportwire.Path, Content: []byte(content)} } +const fleet = `{"schema":"ftw.fleet/1","ftw_version":"v0.146.0-beta.1","channel":"beta","drivers":["pixii","pixii_pv"],"battery_kwh":"5-15","price_zone":"SE4","install_age":"1-6m"}` + func post(s *Service, body []byte, contentType string) *httptest.ResponseRecorder { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/v1/ohttp", bytes.NewReader(body)) @@ -72,7 +76,7 @@ func send(t *testing.T, s *Service, cfg ohttp.KeyConfig, msg []byte) int { t.Fatal(err) } resp, err := ohttp.ParseResponse(plain) - if err != nil || len(plain) != energywire.ResponseSize { + if err != nil || len(plain) != reportwire.ResponseSize { t.Fatalf("inner %x %v", plain, err) } return resp.Status @@ -81,7 +85,7 @@ func send(t *testing.T, s *Service, cfg ohttp.KeyConfig, msg []byte) int { func TestAReportAddsToItsDate(t *testing.T) { s, reg, cfg := newService(t) for range 2 { - if got := send(t, s, cfg, inner(good).Marshal(energywire.RequestSize)); got != http.StatusNoContent { + if got := send(t, s, cfg, inner(good).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { t.Fatalf("status %d", got) } } @@ -91,32 +95,56 @@ func TestAReportAddsToItsDate(t *testing.T) { } } +// A fleet report through the gateway lands in the fleet's day totals, the +// ones /v1/fleet/public reads, and nothing of it in the energy totals. +func TestAFleetReportAddsToTheFleetTotals(t *testing.T) { + s, reg, cfg := newService(t) + if got := send(t, s, cfg, inner(fleet).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("status %d", got) + } + view, err := reg.FleetView(now) + if err != nil || len(view.Days) != 1 || view.Days[0].Reports != 1 || view.Days[0].Zones["SE4"] != 1 { + t.Fatalf("fleet %+v %v", view, err) + } + if totals, _ := reg.EnergyTotals("2026-01-01", "2026-12-31"); len(totals) != 0 { + t.Fatalf("a fleet report reached the energy totals: %v", totals) + } + var r fleetwire.Report + _ = json.Unmarshal([]byte(fleet), &r) + if r.Schema != fleetwire.Schema { + t.Fatal("fixture") + } +} + // Anything but the one padded request with one valid report gets a sealed // 400 and adds nothing. func TestJunkIsRefusedAndAddsNothing(t *testing.T) { s, reg, cfg := newService(t) - pad := energywire.RequestSize + pad := reportwire.RequestSize other := func(f func(*ohttp.Request)) []byte { r := inner(good) f(&r) return r.Marshal(pad) } for name, msg := range map[string][]byte{ - "another path": other(func(r *ohttp.Request) { r.Path = "/v1/fleet" }), - "another method": other(func(r *ohttp.Request) { r.Method = "PUT" }), - "another host": other(func(r *ohttp.Request) { r.Authority = "example.com" }), - "plain http": other(func(r *ohttp.Request) { r.Scheme = "http" }), - "a header": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "x-home", Value: "1"}} }), - "not padded": inner(good).Marshal(0), - "padded more": inner(good).Marshal(pad + 1), - "extra field": inner(strings.TrimSuffix(good, "}") + `,"home":"a"}`).Marshal(pad), - "too much solar": inner(strings.Replace(good, `"pv_generation_kwh":12`, `"pv_generation_kwh":1001`, 1)).Marshal(pad), - "negative": inner(strings.Replace(good, `:12,`, `:-12,`, 1)).Marshal(pad), - "today": inner(strings.Replace(good, "2026-10-23", "2026-10-24", 1)).Marshal(pad), - "four days back": inner(strings.Replace(good, "2026-10-23", "2026-10-20", 1)).Marshal(pad), - "other schema": inner(strings.Replace(good, "ftw.energy/1", "ftw.energy/2", 1)).Marshal(pad), - "a response": ohttp.Response{Status: 200, Content: []byte(good)}.Marshal(pad), - "empty": nil, + "another path": other(func(r *ohttp.Request) { r.Path = "/v1/fleet" }), + "another method": other(func(r *ohttp.Request) { r.Method = "PUT" }), + "another host": other(func(r *ohttp.Request) { r.Authority = "example.com" }), + "plain http": other(func(r *ohttp.Request) { r.Scheme = "http" }), + "a header": other(func(r *ohttp.Request) { r.Fields = []ohttp.Field{{Name: "x-home", Value: "1"}} }), + "not padded": inner(good).Marshal(0), + "padded more": inner(good).Marshal(pad + 1), + "extra field": inner(strings.TrimSuffix(good, "}") + `,"home":"a"}`).Marshal(pad), + "too much solar": inner(strings.Replace(good, `"pv_generation_kwh":12`, `"pv_generation_kwh":1001`, 1)).Marshal(pad), + "negative": inner(strings.Replace(good, `:12,`, `:-12,`, 1)).Marshal(pad), + "today": inner(strings.Replace(good, "2026-10-23", "2026-10-24", 1)).Marshal(pad), + "four days back": inner(strings.Replace(good, "2026-10-23", "2026-10-20", 1)).Marshal(pad), + "other schema": inner(strings.Replace(good, "ftw.energy/1", "ftw.energy/2", 1)).Marshal(pad), + "no schema": inner(`{"date":"2026-10-23"}`).Marshal(pad), + "energy and fleet in one": inner(strings.TrimSuffix(fleet, "}") + `,"pv_generation_kwh":12}`).Marshal(pad), + "bad fleet": inner(strings.Replace(fleet, `"SE4"`, `"myhouse"`, 1)).Marshal(pad), + "a response": ohttp.Response{Status: 200, Content: []byte(good)}.Marshal(pad), + "empty": nil, } { if got := send(t, s, cfg, msg); got != http.StatusBadRequest { t.Fatalf("%s: %d", name, got) @@ -148,12 +176,15 @@ func TestJunkIsRefusedAndAddsNothing(t *testing.T) { if totals, _ := reg.EnergyTotals("2026-01-01", "2026-12-31"); len(totals) != 0 { t.Fatalf("junk reached the totals: %v", totals) } + if view, _ := reg.FleetView(now); len(view.Days) != 0 { + t.Fatalf("junk reached the fleet totals: %+v", view) + } } // One budget for all requests, since they all come from the relay. func TestRequestsShareOneBudget(t *testing.T) { s, _, cfg := newService(t) - sealed, _, _ := ohttp.Encapsulate(cfg, inner(good).Marshal(energywire.RequestSize)) + sealed, _, _ := ohttp.Encapsulate(cfg, inner(good).Marshal(reportwire.RequestSize)) for i := range int(burst) { if w := post(s, sealed, ohttp.RequestType); w.Code == http.StatusTooManyRequests { t.Fatalf("limited after %d", i) diff --git a/internal/server/energy_e2e_test.go b/internal/server/reports_e2e_test.go similarity index 63% rename from internal/server/energy_e2e_test.go rename to internal/server/reports_e2e_test.go index 5a45a3f..cab0e97 100644 --- a/internal/server/energy_e2e_test.go +++ b/internal/server/reports_e2e_test.go @@ -11,7 +11,9 @@ import ( "time" "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" "github.com/srcfl/ftw/go/protocol/ohttp" + "github.com/srcfl/ftw/go/protocol/reportwire" ) func testGateway(t *testing.T) *ohttp.Gateway { @@ -27,11 +29,12 @@ func testGateway(t *testing.T) *ohttp.Gateway { return g } -// Box to relay to gateway to totals. The box half is what Core sends: the -// key configuration from /.well-known/ohttp-gateway, the one inner request, -// padded. The relay forwards only the body and its type, and never sees the -// report. ftwcloudd adds it to the date's totals and answers a sealed 204. -func TestAnEnergyReportReachesTheTotalsThroughARelay(t *testing.T) { +// Box to relay to gateway to totals, for both reports. The box half is what +// Core sends: the key configuration from /.well-known/ohttp-gateway, each +// report alone in the one padded inner request. The relay forwards only the +// body and its type, never sees a report, and sees one length. ftwcloudd adds +// each to its own day totals and answers a sealed 204. +func TestBothReportsReachTheirTotalsThroughARelay(t *testing.T) { s, _ := newServer(t) cloud := httptest.NewServer(s.Handler()) defer cloud.Close() @@ -65,9 +68,14 @@ func TestAnEnergyReportReachesTheTotalsThroughARelay(t *testing.T) { yesterday := time.Now().UTC().AddDate(0, 0, -1).Format(time.DateOnly) report := energywire.Report{Schema: energywire.Schema, Date: yesterday, PVGeneration: 12, GridImport: 30, GridExport: 4, BatteryCharge: 9, BatteryDischarge: 8} - for i := range 3 { - body, _ := json.Marshal(report) - inner := ohttp.Request{Method: energywire.Method, Scheme: energywire.Scheme, Authority: energywire.Authority, Path: energywire.Path, Content: body}.Marshal(energywire.RequestSize) + energyBody, _ := json.Marshal(report) + fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"}) + // Three energy reports and two fleet reports, each alone. + for i, body := range [][]byte{energyBody, fleetBody, energyBody, fleetBody, energyBody} { + inner, ok := reportwire.Request(body) + if !ok { + t.Fatal("report does not fit") + } sealed, reader, err := ohttp.Encapsulate(configs[0], inner) if err != nil { t.Fatal(err) @@ -85,10 +93,10 @@ func TestAnEnergyReportReachesTheTotalsThroughARelay(t *testing.T) { if err != nil { t.Fatal(err) } - if got, err := ohttp.ParseResponse(msg); err != nil || got.Status != http.StatusNoContent || len(msg) != energywire.ResponseSize { + if got, err := ohttp.ParseResponse(msg); err != nil || got.Status != http.StatusNoContent || len(msg) != reportwire.ResponseSize { t.Fatalf("report %d: %+v %v", i, got, err) } - if bytes.Contains(seen[i], []byte("kwh")) || bytes.Contains(seen[i], []byte(yesterday)) { + if bytes.Contains(seen[i], []byte("kwh")) || bytes.Contains(seen[i], []byte(yesterday)) || bytes.Contains(seen[i], []byte("sungrow")) { t.Fatal("the relay saw the report") } } @@ -97,7 +105,14 @@ func TestAnEnergyReportReachesTheTotalsThroughARelay(t *testing.T) { if err != nil || day.Reports != 3 || day.Sums != [6]int64{36, 90, 12, 27, 24, 0} { t.Fatalf("totals %+v %v", day, err) } - if len(seen[0]) != len(seen[1]) { - t.Fatal("two reports differ in length on the relay") + for _, b := range seen { + if len(b) != len(seen[0]) { + t.Fatal("reports differ in length on the relay") + } + } + // The fleet reports land where /v1/fleet/public reads them. + view, err := s.Registry.FleetView(time.Now()) + if err != nil || len(view.Days) != 1 || view.Days[0].Reports != 2 || view.Days[0].Drivers["sungrow"] != 2 { + t.Fatalf("fleet %+v %v", view, err) } } diff --git a/internal/server/server.go b/internal/server/server.go index 3c5ca5e..aa50ba8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -5,7 +5,7 @@ // /v1/app a phone's Noise session to the service (package appsession) // /v1/app/once a watch's one-shot read, Noise inside one POST (package appsession) // /v1/now a widget's token read (package token) -// /v1/ohttp the Oblivious HTTP gateway for anonymous daily energy reports (package energy) +// /v1/ohttp the Oblivious HTTP gateway for the anonymous daily reports (package gateway) // /.well-known/ohttp-gateway that gateway's key configuration // /v1/fleet, /v1/fleet/public, /v1/energy/public anonymous daily totals // /source the source of the running service, as AGPL-3.0 asks @@ -29,7 +29,7 @@ import ( "time" "github.com/srcfl/ftw-cloud/internal/appsession" - "github.com/srcfl/ftw-cloud/internal/energy" + "github.com/srcfl/ftw-cloud/internal/gateway" "github.com/srcfl/ftw-cloud/internal/oauth" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/store" @@ -63,7 +63,7 @@ type Server struct { // they call. Nil serves none. OAuth *oauth.Server Fleet http.Handler - Energy *energy.Service + Gateway *gateway.Service Statistics http.Handler MCP http.Handler Log *slog.Logger @@ -106,10 +106,10 @@ func (s *Server) Handler() http.Handler { } else { mux.HandleFunc("/v1/fleet", notYet) } - if s.Energy != nil { - mux.Handle("/v1/ohttp", s.Energy) - mux.HandleFunc("/.well-known/ohttp-gateway", s.Energy.Keys) - mux.HandleFunc("/v1/energy/public", s.Energy.Public) + if s.Gateway != nil { + mux.Handle("/v1/ohttp", s.Gateway) + mux.HandleFunc("/.well-known/ohttp-gateway", s.Gateway.Keys) + mux.HandleFunc("/v1/energy/public", s.Gateway.Public) } else { mux.HandleFunc("/v1/ohttp", notYet) } @@ -210,8 +210,8 @@ func (s *Server) Upkeep(ctx context.Context, every time.Duration) { s.Log.Warn("fleet upkeep failed") } } - if s.Energy != nil { - if err := s.Energy.Prune(time.Now()); err != nil { + if s.Gateway != nil { + if err := s.Gateway.Prune(time.Now()); err != nil { s.Log.Warn("energy upkeep failed") } } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 9e1e01d..b3a5a43 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -14,8 +14,8 @@ import ( "testing" "github.com/srcfl/ftw-cloud/internal/appsession" - "github.com/srcfl/ftw-cloud/internal/energy" "github.com/srcfl/ftw-cloud/internal/fleet" + "github.com/srcfl/ftw-cloud/internal/gateway" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/statistics" "github.com/srcfl/ftw-cloud/internal/store" @@ -48,7 +48,7 @@ func newServer(t *testing.T) (*Server, string) { if err != nil { t.Fatal(err) } - if s.Energy, err = energy.New(reg, testGateway(t)); err != nil { + if s.Gateway, err = gateway.New(reg, testGateway(t)); err != nil { t.Fatal(err) } up.Statistics = true From 79caea0a9adcfbd4f7091560df35af4ce0a93be6 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 08:59:43 +0200 Subject: [PATCH 3/8] fix(gateway): withhold an energy sum fewer than five reports had any of A date with five reports and one car published that car's whole charge in vehicle_charge_kwh, because the sum counted the other four reports' zeros. energy_days now counts, for each sum, the reports that had any of it, and /v1/energy/public shows a sum only when none or at least five had some. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++-- internal/gateway/gateway.go | 6 +++-- internal/gateway/gateway_test.go | 10 ++++++++ internal/registry/energy.go | 43 +++++++++++++++++++++++++------- internal/registry/energy_test.go | 4 +-- 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 642f51a..98dc5ef 100644 --- a/README.md +++ b/README.md @@ -551,8 +551,11 @@ operator and for anyone checking the pin. `GET /v1/energy/public` lists the closed dates of the last 30 days, newest first: the report count and the six sums. A date appears only once no report may name it any more, so two readings never differ by one report. A date with -fewer than 5 reports shows null, as `/v1/fleet/public` does. Rows go after 30 -days. These are reports, not homes. +fewer than 5 reports shows null, as `/v1/fleet/public` does. So does a sum that +between one and four of them had any of: five reports with one car is that +car's charge. A sum nobody had any of shows 0. Rows, which hold for each sum +how many reports had any of it, go after 30 days. These are reports, not +homes. Limits, plainly. A report cannot prove that a real box sent it, and the relay could replay one it forwarded; the bounds and the caps keep both to diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 781b516..21f63ab 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -166,7 +166,9 @@ func (s *Service) Keys(w http.ResponseWriter, r *http.Request) { // Public is GET /v1/energy/public: the closed dates of the last EnergyDays, // newest first. A date shows only once no report may name it any more, so // two readings never differ by one report. A date with fewer than Minimum -// reports shows null. +// reports shows null, and so does a sum that between one and Minimum-1 of +// them had any of: five reports with one car is that car's charge. A sum +// nobody had any of shows 0. func (s *Service) Public(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "public, max-age=60") if r.Method != http.MethodGet { @@ -187,7 +189,7 @@ func (s *Service) Public(w http.ResponseWriter, r *http.Request) { day, row := totals[date], map[string]any{"date": date, "reports": nil} for i, name := range energywire.Fields { row[name] = nil - if day.Reports >= Minimum { + if day.Reports >= Minimum && (day.Had[i] == 0 || day.Had[i] >= Minimum) { row[name] = day.Sums[i] } } diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 7c973c0..a95fd46 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -222,6 +222,13 @@ func TestAPublicDateIsClosedAndHoldsFiveReports(t *testing.T) { } add("2026-10-18", 5) add("2026-10-19", 4) + // Five reports, one of them with a car: that sum is that car's. + add("2026-10-17", 4) + var car energywire.Report + _ = json.Unmarshal([]byte(strings.Replace(strings.Replace(good, "2026-10-23", "2026-10-17", 1), `"vehicle_charge_kwh":0`, `"vehicle_charge_kwh":20`, 1)), &car) + if ok, err := reg.EnergyAdd(car); err != nil || !ok { + t.Fatal(err) + } add("2026-10-23", 9) // still open for reports add("2026-09-23", 9) // past the window @@ -247,6 +254,9 @@ func TestAPublicDateIsClosedAndHoldsFiveReports(t *testing.T) { if d := byDate["2026-10-18"]; d["reports"] != 5.0 || d["pv_generation_kwh"] != 60.0 || d["vehicle_charge_kwh"] != 0.0 { t.Fatalf("five reports: %v", d) } + if d := byDate["2026-10-17"]; d["reports"] != 5.0 || d["pv_generation_kwh"] != 60.0 || d["vehicle_charge_kwh"] != nil { + t.Fatalf("one car among five: %v", d) + } for _, date := range []string{"2026-10-19", "2026-10-20"} { for k, v := range byDate[date] { if k != "date" && v != nil { diff --git a/internal/registry/energy.go b/internal/registry/energy.go index af48fd8..9b92e07 100644 --- a/internal/registry/energy.go +++ b/internal/registry/energy.go @@ -9,8 +9,9 @@ import ( // EnergyDays is how long a date's energy totals are kept. const EnergyDays = 30 -// EnergyInit creates only daily totals: one row per date, a count and six -// sums. No report, address or time of arrival goes on disk. +// EnergyInit creates only daily totals: one row per date, a count, six sums, +// and for each sum how many reports had any of it. No report, address or time +// of arrival goes on disk. func (r *Registry) EnergyInit() error { _, err := r.db.Exec(`CREATE TABLE IF NOT EXISTS energy_days ( date TEXT PRIMARY KEY, @@ -20,7 +21,13 @@ func (r *Registry) EnergyInit() error { grid_export_kwh INTEGER NOT NULL, battery_charge_kwh INTEGER NOT NULL, battery_discharge_kwh INTEGER NOT NULL, - vehicle_charge_kwh INTEGER NOT NULL)`) + vehicle_charge_kwh INTEGER NOT NULL, + pv_generation_reports INTEGER NOT NULL, + grid_import_reports INTEGER NOT NULL, + grid_export_reports INTEGER NOT NULL, + battery_charge_reports INTEGER NOT NULL, + battery_discharge_reports INTEGER NOT NULL, + vehicle_charge_reports INTEGER NOT NULL)`) return err } @@ -28,16 +35,28 @@ func (r *Registry) EnergyInit() error { // means the date already holds energywire.MaxReports. func (r *Registry) EnergyAdd(rep energywire.Report) (bool, error) { s := rep.Sums() - res, err := r.db.Exec(`INSERT INTO energy_days VALUES (?, 1, ?, ?, ?, ?, ?, ?) + var had [6]int + for i, v := range s { + if v > 0 { + had[i] = 1 + } + } + res, err := r.db.Exec(`INSERT INTO energy_days VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(date) DO UPDATE SET reports = reports + 1, pv_generation_kwh = pv_generation_kwh + excluded.pv_generation_kwh, grid_import_kwh = grid_import_kwh + excluded.grid_import_kwh, grid_export_kwh = grid_export_kwh + excluded.grid_export_kwh, battery_charge_kwh = battery_charge_kwh + excluded.battery_charge_kwh, battery_discharge_kwh = battery_discharge_kwh + excluded.battery_discharge_kwh, - vehicle_charge_kwh = vehicle_charge_kwh + excluded.vehicle_charge_kwh + vehicle_charge_kwh = vehicle_charge_kwh + excluded.vehicle_charge_kwh, + pv_generation_reports = pv_generation_reports + excluded.pv_generation_reports, + grid_import_reports = grid_import_reports + excluded.grid_import_reports, + grid_export_reports = grid_export_reports + excluded.grid_export_reports, + battery_charge_reports = battery_charge_reports + excluded.battery_charge_reports, + battery_discharge_reports = battery_discharge_reports + excluded.battery_discharge_reports, + vehicle_charge_reports = vehicle_charge_reports + excluded.vehicle_charge_reports WHERE reports < ?`, - rep.Date, s[0], s[1], s[2], s[3], s[4], s[5], energywire.MaxReports) + rep.Date, s[0], s[1], s[2], s[3], s[4], s[5], had[0], had[1], had[2], had[3], had[4], had[5], energywire.MaxReports) if err != nil { return false, err } @@ -45,17 +64,22 @@ func (r *Registry) EnergyAdd(rep energywire.Report) (bool, error) { return n == 1, err } -// EnergyDay is one date's totals, sums in energywire.Fields order. +// EnergyDay is one date's totals, sums in energywire.Fields order. Had counts, +// for each sum, the reports with any of it: a sum of five reports that only +// one had any of is that one home's. type EnergyDay struct { Date string Reports int64 Sums [6]int64 + Had [6]int64 } // EnergyTotals reads the dates from first to last, both included. func (r *Registry) EnergyTotals(first, last string) (map[string]EnergyDay, error) { rows, err := r.rd.Query(`SELECT date, reports, pv_generation_kwh, grid_import_kwh, grid_export_kwh, - battery_charge_kwh, battery_discharge_kwh, vehicle_charge_kwh + battery_charge_kwh, battery_discharge_kwh, vehicle_charge_kwh, + pv_generation_reports, grid_import_reports, grid_export_reports, + battery_charge_reports, battery_discharge_reports, vehicle_charge_reports FROM energy_days WHERE date >= ? AND date <= ? LIMIT ?`, first, last, EnergyDays+1) if err != nil { return nil, err @@ -64,7 +88,8 @@ func (r *Registry) EnergyTotals(first, last string) (map[string]EnergyDay, error out := map[string]EnergyDay{} for rows.Next() { var d EnergyDay - if err := rows.Scan(&d.Date, &d.Reports, &d.Sums[0], &d.Sums[1], &d.Sums[2], &d.Sums[3], &d.Sums[4], &d.Sums[5]); err != nil { + if err := rows.Scan(&d.Date, &d.Reports, &d.Sums[0], &d.Sums[1], &d.Sums[2], &d.Sums[3], &d.Sums[4], &d.Sums[5], + &d.Had[0], &d.Had[1], &d.Had[2], &d.Had[3], &d.Had[4], &d.Had[5]); err != nil { return nil, err } out[d.Date] = d diff --git a/internal/registry/energy_test.go b/internal/registry/energy_test.go index 61324ac..0a17e3a 100644 --- a/internal/registry/energy_test.go +++ b/internal/registry/energy_test.go @@ -30,11 +30,11 @@ func TestEnergyKeepsOnlyDailyTotals(t *testing.T) { if err := r.db.QueryRow(`SELECT COUNT(*) FROM energy_days`).Scan(&rows); err != nil || rows != 2 { t.Fatalf("rows %d %v", rows, err) } - if err := r.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('energy_days')`).Scan(&columns); err != nil || columns != 2+len(energywire.Fields) { + if err := r.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('energy_days')`).Scan(&columns); err != nil || columns != 2+2*len(energywire.Fields) { t.Fatalf("columns %d %v", columns, err) } totals, err := r.EnergyTotals("2026-10-22", "2026-10-23") - if d := totals["2026-10-23"]; err != nil || d.Reports != 5 || d.Sums[0] != 35 || d.Sums[1] != 5 { + if d := totals["2026-10-23"]; err != nil || d.Reports != 5 || d.Sums[0] != 35 || d.Sums[1] != 5 || d.Had != [6]int64{5, 5, 0, 0, 0, 0} { t.Fatalf("%+v %v", d, err) } From 0c348b2aa3c5bff09a7b7c6662153fd43dedd816 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:13:37 +0200 Subject: [PATCH 4/8] fix(gateway): count each report once, budget per kind, close the fleet view From review of #50: - Replays: each inner request now carries a random report id and its send day (srcfl/ftw a7c025e8). The gateway refuses a request sent before yesterday and keeps a hash of each id per day for three days, in a WITHOUT ROWID table with no time, so a replay or a retry adds nothing and is still answered 204. Seven posts of one ciphertext now count once. - /v1/fleet/public lists closed UTC days only, so the relay cannot poll it and match a new count to the report it just forwarded. - Each report kind has its own budget, spent only by a report that opened and decoded; junk no longer silences real reports. - A 10-second deadline on reading the body of /v1/ohttp and /v1/fleet; a failed read closes the connection. - ohttp keys: data after the last whole PEM block, such as a second key cut short in a rotation, is refused instead of silently ignored. Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++++-- cmd/ftwcloudd/ohttp.go | 6 ++ cmd/ftwcloudd/ohttp_test.go | 10 +++ go.mod | 2 +- go.sum | 2 + internal/fleet/fleet.go | 17 ++++- internal/fleet/fleet_test.go | 32 +++++++++ internal/fleet/public.go | 8 ++- internal/fleet/public_test.go | 6 ++ internal/gateway/gateway.go | 96 +++++++++++++++++-------- internal/gateway/gateway_test.go | 105 ++++++++++++++++++++++++---- internal/registry/energy.go | 12 +++- internal/registry/fleet.go | 11 ++- internal/registry/reports.go | 66 +++++++++++++++++ internal/server/reports_e2e_test.go | 5 +- 15 files changed, 349 insertions(+), 54 deletions(-) create mode 100644 internal/registry/reports.go diff --git a/README.md b/README.md index 98dc5ef..3fffd73 100644 --- a/README.md +++ b/README.md @@ -542,9 +542,15 @@ the box's address and a ciphertext of fixed length (1,591 bytes). address. It reads the one padded inner request, chooses the decoder by schema, adds the report to its own day totals and answers a sealed 204, or 400 for anything else: exact fields, and for energy at most 1,000 kWh each -and a date in the last three closed days. It keeps nothing per report: no -row, logged body or address. All requests share one rate budget, and a day -takes at most 100,000 reports of each kind. +and a date in the last three closed days. Each request carries a random +report id and the UTC date it was sent. The gateway refuses one sent before +yesterday, and keeps a hash of each id under its date for three days, in a +table with no time or arrival order, so a replay or a retry counts once and +is still answered 204 (RFC 9458, section 6.5). Beyond that it keeps nothing +per report: no row, logged body or address. Each kind has its own rate +budget, spent only by a report that opened and decoded, so junk cannot +silence real reports. A day takes at most 100,000 reports of each kind, and +a body has 10 seconds to arrive. `GET /.well-known/ohttp-gateway` serves the key configuration for the relay operator and for anyone checking the pin. @@ -557,9 +563,16 @@ car's charge. A sum nobody had any of shows 0. Rows, which hold for each sum how many reports had any of it, go after 30 days. These are reports, not homes. -Limits, plainly. A report cannot prove that a real box sent it, and the -relay could replay one it forwarded; the bounds and the caps keep both to -noise. Privacy Pass-style tokens are the later fix and are not built. The +`/v1/fleet/public` also lists closed days only, so whoever forwarded a +report, the relay above all, cannot poll the view and see which count it +moved. + +Limits, plainly. A report cannot prove that a real box sent it. Anyone with +the public key configuration can post reports, and the relay can drop some. +One real report and four all-zero fakes would publish that home's six sums +under "5 reports". The floor of five counts reports, not boxes. This is an +open design question for Fredrik; bounds and caps do not solve it. +Privacy Pass-style tokens would; they are not built. The relay and Sourceful together could join reports to addresses by time. This service also holds readings from homes with the readable copy on, including 48 hours of history tiles, from which the same daily sums could be computed; diff --git a/cmd/ftwcloudd/ohttp.go b/cmd/ftwcloudd/ohttp.go index 0489bb9..43718ed 100644 --- a/cmd/ftwcloudd/ohttp.go +++ b/cmd/ftwcloudd/ohttp.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "crypto/ecdh" "crypto/sha256" "crypto/x509" @@ -51,6 +52,11 @@ func loadOHTTPKeys(path string) (*ohttp.Gateway, error) { } keys = append(keys, ohttp.Key{ID: ohttpKeyID(x.PublicKey().Bytes()), Private: x}) } + // A block cut short, as a rotation pasted in part would leave it, must + // not leave the new key quietly unloaded. + if len(bytes.TrimSpace(raw)) > 0 { + return nil, fmt.Errorf("ohttp key: %s has data after its last whole key", path) + } if len(keys) == 0 { return nil, fmt.Errorf("ohttp key: %s holds no key", path) } diff --git a/cmd/ftwcloudd/ohttp_test.go b/cmd/ftwcloudd/ohttp_test.go index 6126066..b20c3d2 100644 --- a/cmd/ftwcloudd/ohttp_test.go +++ b/cmd/ftwcloudd/ohttp_test.go @@ -56,6 +56,16 @@ func TestAGatewayKeyLoadsAsThePinSays(t *testing.T) { if pins := ohttpPins(nil); pins != "" { t.Fatal(pins) } + + // A second block cut short, as a rotation pasted in part leaves it, is + // refused rather than quietly left unloaded. + cut := filepath.Join(dir, "cut.key") + if err := os.WriteFile(cut, append(a, b[:len(b)/2]...), 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadOHTTPKeys(cut); err == nil || !strings.Contains(err.Error(), "after its last whole key") { + t.Fatalf("a cut second key: %v", err) + } } func TestOnlyX25519GatewayKeysLoad(t *testing.T) { diff --git a/go.mod b/go.mod index 0a43b22..1c93c99 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95 + github.com/srcfl/ftw/go/protocol v0.0.0-20261024070811-a7c025e87221 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 8918c5c..0ce9f8f 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242 h1:ePRqHUvSp github.com/srcfl/ftw/go/protocol v0.0.0-20261024055601-28fddd802242/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95 h1:VMpQKeK7HImzVWeTNIuo2cDl45WFWhFttR/X+hYDOY0= github.com/srcfl/ftw/go/protocol v0.0.0-20261024063545-4cfd65f7da95/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024070811-a7c025e87221 h1:KN4wy7SnAj8x2B+atAQvVVgtklj4io6Y3iLxhoJve94= +github.com/srcfl/ftw/go/protocol v0.0.0-20261024070811-a7c025e87221/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/fleet/fleet.go b/internal/fleet/fleet.go index 1c34c88..664d789 100644 --- a/internal/fleet/fleet.go +++ b/internal/fleet/fleet.go @@ -3,6 +3,7 @@ package fleet import ( "encoding/json" + "errors" "io" "net/http" "sync" @@ -12,6 +13,9 @@ import ( "github.com/srcfl/ftw/go/protocol/fleetwire" ) +// bodyDeadline bounds reading a report. +var bodyDeadline = 10 * time.Second + type Service struct { reg *registry.Registry now func() time.Time @@ -47,11 +51,22 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTooManyRequests) return } + // A slow sender cannot hold the connection open. + rc := http.NewResponseController(w) + _ = rc.SetReadDeadline(time.Now().Add(bodyDeadline)) data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, fleetwire.MaxBodyBytes)) if err != nil { - w.WriteHeader(http.StatusRequestEntityTooLarge) + // The rest of the body is never read: close rather than wait for it. + w.Header().Set("Connection", "close") + var large *http.MaxBytesError + if errors.As(err, &large) { + w.WriteHeader(http.StatusRequestEntityTooLarge) + } else { + w.WriteHeader(http.StatusRequestTimeout) + } return } + _ = rc.SetReadDeadline(time.Time{}) report, err := fleetwire.Decode(data) if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/internal/fleet/fleet_test.go b/internal/fleet/fleet_test.go index 4ee230a..42af44c 100644 --- a/internal/fleet/fleet_test.go +++ b/internal/fleet/fleet_test.go @@ -1,10 +1,13 @@ package fleet import ( + "bufio" "bytes" "encoding/json" + "fmt" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw/go/protocol/fleetwire" + "net" "net/http" "net/http/httptest" "path/filepath" @@ -90,3 +93,32 @@ func TestReportsBecomeOnlyBoundedDailyTotalsAcrossARestart(t *testing.T) { } // The only extra table holds date and aggregate JSON, not source addresses. } + +// A sender that stops halfway through its body is cut off. +func TestASlowFleetBodyIsCutOff(t *testing.T) { + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) + if err != nil { + t.Fatal(err) + } + defer reg.Close() + svc, err := New(reg) + if err != nil { + t.Fatal(err) + } + old := bodyDeadline + bodyDeadline = 200 * time.Millisecond + t.Cleanup(func() { bodyDeadline = old }) + srv := httptest.NewServer(svc) + defer srv.Close() + conn, err := net.Dial("tcp", srv.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _, _ = fmt.Fprintf(conn, "POST /v1/fleet HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n{") + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil || resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("slow body: %v %v", resp, err) + } +} diff --git a/internal/fleet/public.go b/internal/fleet/public.go index e12ea63..1b879b9 100644 --- a/internal/fleet/public.go +++ b/internal/fleet/public.go @@ -9,7 +9,9 @@ import ( ) // Public exports only cells already large enough for the project dashboard. -// A day below the minimum has a null count and no dimension labels. +// A day below the minimum has a null count and no dimension labels. Only +// closed UTC days are listed: today's running counts would let whoever just +// forwarded a report, such as the relay, see which count it moved. func (s *Service) Public(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "public, max-age=60") if r.Method != http.MethodGet { @@ -24,8 +26,12 @@ func (s *Service) Public(w http.ResponseWriter, r *http.Request) { // Five since 23 October 2026 (Fredrik): the fleet is small and the // reports carry no home id. A day and each label below it stay hidden. const minimum = 5 + today := s.now().UTC().Format(time.DateOnly) days := make([]map[string]any, 0, len(view.Days)) for _, d := range view.Days { + if d.Date >= today { + continue + } row := map[string]any{"date": d.Date, "reports": nil} for key, values := range map[string]map[string]int{"ftw_versions": d.Versions, "channels": d.Channels, "drivers": d.Drivers, "battery_kwh": d.Battery, "price_zones": d.Zones, "install_age": d.Age} { safe := map[string]int{} diff --git a/internal/fleet/public_test.go b/internal/fleet/public_test.go index ec90ea4..b618ddd 100644 --- a/internal/fleet/public_test.go +++ b/internal/fleet/public_test.go @@ -39,6 +39,12 @@ func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) { } return v } + // Today is still open, so its running counts are not listed: whoever + // forwarded a report could see which count it moved. + if days := get()["days"].([]any); len(days) != 0 { + t.Fatalf("today listed: %v", days) + } + svc.now = func() time.Time { return now.Add(24 * time.Hour) } v := get() d := v["days"].([]any)[0].(map[string]any) if d["reports"] != nil || len(d["ftw_versions"].(map[string]any)) != 0 { diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 21f63ab..e911a8e 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -14,6 +14,7 @@ package gateway import ( "encoding/json" + "errors" "io" "mime" "net/http" @@ -33,12 +34,18 @@ const ( maxRequest = 2 << 10 // Minimum is the group size for a public date, as /v1/fleet/public's. Minimum = 5 - // rate and burst are one shared budget for every request. All of them - // arrive from the relay, so there is no address worth keeping apart. + // rate and burst are each report kind's budget, spent only by a report + // that opened and decoded, so junk cannot use up the real reports'. + // Every request arrives from the relay, so there is no address worth + // keeping apart. rate = 5.0 burst = 120.0 ) +// bodyDeadline bounds reading a request body, so a slow sender cannot hold a +// connection open. +var bodyDeadline = 10 * time.Second + type Service struct { reg *registry.Registry // gw is nil on a host without a gateway key; the gateway routes then say @@ -47,6 +54,11 @@ type Service struct { keys []byte now func() time.Time + fleet, energy budget +} + +// budget is a token bucket. +type budget struct { mu sync.Mutex at time.Time tokens float64 @@ -59,10 +71,14 @@ func New(reg *registry.Registry, gw *ohttp.Gateway) (*Service, error) { if err := reg.EnergyInit(); err != nil { return nil, err } - if err := reg.EnergyPrune(time.Now()); err != nil { + if err := reg.ReportsInit(); err != nil { + return nil, err + } + s := &Service{reg: reg, gw: gw, now: time.Now} + s.fleet.tokens, s.energy.tokens = burst, burst + if err := s.Prune(time.Now()); err != nil { return nil, err } - s := &Service{reg: reg, gw: gw, now: time.Now, tokens: burst} if gw != nil { s.keys = ohttp.MarshalKeys(gw.Configs()...) } @@ -71,7 +87,8 @@ func New(reg *registry.Registry, gw *ohttp.Gateway) (*Service, error) { // ServeHTTP is the gateway, POST /v1/ohttp. A request it cannot open gets a // plain 400. One it opens gets a sealed answer: 204 when the report is -// added, 400 when it is not one valid report, 429 when its day is full. +// added or was counted already, 400 when it is not one valid report, 429 +// when its kind's budget or its day is full. func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") if s.gw == nil { @@ -87,15 +104,21 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnsupportedMediaType) return } - if !s.allow() { - w.WriteHeader(http.StatusTooManyRequests) - return - } + rc := http.NewResponseController(w) + _ = rc.SetReadDeadline(time.Now().Add(bodyDeadline)) body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxRequest)) if err != nil { - w.WriteHeader(http.StatusRequestEntityTooLarge) + // The rest of the body is never read: close rather than wait for it. + w.Header().Set("Connection", "close") + var large *http.MaxBytesError + if errors.As(err, &large) { + w.WriteHeader(http.StatusRequestEntityTooLarge) + } else { + w.WriteHeader(http.StatusRequestTimeout) + } return } + _ = rc.SetReadDeadline(time.Time{}) msg, sealer, err := s.gw.Decapsulate(body) if err != nil { w.WriteHeader(http.StatusBadRequest) @@ -111,33 +134,43 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // add reads the one request the gateway answers and adds its report to the -// day totals of its kind. +// day totals of its kind, once: a report whose id was counted already is +// answered 204 and adds nothing, so a replay or a retry is harmless. func (s *Service) add(msg []byte) int { - body, ok := reportwire.Body(msg) - if !ok { + now := s.now() + report, ok := reportwire.Read(msg) + if !ok || !reportwire.Fresh(report.Day, now) { return http.StatusBadRequest } - var added bool + var added, repeat bool var err error - switch reportwire.Schema(body) { + switch reportwire.Schema(report.Body) { case fleetwire.Schema: - report, decodeErr := fleetwire.Decode(body) + fleet, decodeErr := fleetwire.Decode(report.Body) if decodeErr != nil { return http.StatusBadRequest } - added, err = s.reg.FleetPut(report, s.now()) + if !s.fleet.allow(now) { + return http.StatusTooManyRequests + } + added, repeat, err = s.reg.FleetPutOnce(report.Day, report.ID, fleet, now) case energywire.Schema: - report, decodeErr := energywire.Decode(body, s.now()) + energy, decodeErr := energywire.Decode(report.Body, now) if decodeErr != nil { return http.StatusBadRequest } - added, err = s.reg.EnergyAdd(report) + if !s.energy.allow(now) { + return http.StatusTooManyRequests + } + added, repeat, err = s.reg.EnergyAddOnce(report.Day, report.ID, energy) default: return http.StatusBadRequest } switch { case err != nil: return http.StatusServiceUnavailable + case repeat: + return http.StatusNoContent case !added: return http.StatusTooManyRequests } @@ -203,19 +236,24 @@ func (s *Service) Public(w http.ResponseWriter, r *http.Request) { "observed_at": now.Format(time.RFC3339), "minimum": Minimum, "days": days}) } -func (s *Service) Prune(now time.Time) error { return s.reg.EnergyPrune(now) } +// Prune drops energy totals and report ids past their days. +func (s *Service) Prune(now time.Time) error { + if err := s.reg.EnergyPrune(now); err != nil { + return err + } + return s.reg.ReportsPrune(now) +} -func (s *Service) allow() bool { - s.mu.Lock() - defer s.mu.Unlock() - now := s.now() - if !s.at.IsZero() { - s.tokens = min(burst, s.tokens+max(0, now.Sub(s.at).Seconds())*rate) +func (b *budget) allow(now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + if !b.at.IsZero() { + b.tokens = min(burst, b.tokens+max(0, now.Sub(b.at).Seconds())*rate) } - s.at = now - if s.tokens < 1 { + b.at = now + if b.tokens < 1 { return false } - s.tokens-- + b.tokens-- return true } diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index a95fd46..b25f54c 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -1,9 +1,14 @@ package gateway import ( + "bufio" "bytes" "crypto/ecdh" + "crypto/rand" + "encoding/base64" "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" "path/filepath" @@ -45,8 +50,21 @@ func newService(t *testing.T) (*Service, *registry.Registry, ohttp.KeyConfig) { return s, reg, gw.Configs()[0] } +// inner is one report as a box sends it: a fresh id, sent today. func inner(content string) ohttp.Request { - return ohttp.Request{Method: reportwire.Method, Scheme: reportwire.Scheme, Authority: reportwire.Authority, Path: reportwire.Path, Content: []byte(content)} + return innerWith(content, newID(), "2026-10-24") +} + +func innerWith(content string, id []byte, day string) ohttp.Request { + return ohttp.Request{Method: reportwire.Method, Scheme: reportwire.Scheme, Authority: reportwire.Authority, Path: reportwire.Path, + Fields: []ohttp.Field{{Name: "report-id", Value: base64.RawURLEncoding.EncodeToString(id)}, {Name: "report-day", Value: day}}, + Content: []byte(content)} +} + +func newID() []byte { + id := make([]byte, reportwire.IDBytes) + _, _ = rand.Read(id) + return id } const fleet = `{"schema":"ftw.fleet/1","ftw_version":"v0.146.0-beta.1","channel":"beta","drivers":["pixii","pixii_pv"],"battery_kwh":"5-15","price_zone":"SE4","install_age":"1-6m"}` @@ -181,22 +199,85 @@ func TestJunkIsRefusedAndAddsNothing(t *testing.T) { } } -// One budget for all requests, since they all come from the relay. -func TestRequestsShareOneBudget(t *testing.T) { - s, _, cfg := newService(t) +// A replayed request, or a retry of one report, counts once and is still +// answered 204, so neither the relay nor a lost answer moves a total. +func TestAReportCountsOnceHoweverOftenItArrives(t *testing.T) { + s, reg, cfg := newService(t) sealed, _, _ := ohttp.Encapsulate(cfg, inner(good).Marshal(reportwire.RequestSize)) + for i := range 7 { + w := post(s, sealed, ohttp.RequestType) + if w.Code != http.StatusOK { + t.Fatalf("replay %d: %d", i, w.Code) + } + } + id := newID() + for range 3 { + if got := send(t, s, cfg, innerWith(fleet, id, "2026-10-24").Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("retry: %d", got) + } + } + totals, _ := reg.EnergyTotals("2026-10-23", "2026-10-23") + view, _ := reg.FleetView(now) + if totals["2026-10-23"].Reports != 1 || len(view.Days) != 1 || view.Days[0].Reports != 1 { + t.Fatalf("energy %+v fleet %+v", totals, view) + } +} + +// A request sent before yesterday is refused, so a captured one cannot be +// posted again once its id is forgotten. Yesterday's still counts. +func TestAnOldRequestIsRefused(t *testing.T) { + s, _, cfg := newService(t) + if got := send(t, s, cfg, innerWith(fleet, newID(), "2026-10-22").Marshal(reportwire.RequestSize)); got != http.StatusBadRequest { + t.Fatalf("two days old: %d", got) + } + if got := send(t, s, cfg, innerWith(fleet, newID(), "2026-10-23").Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("yesterday: %d", got) + } +} + +// Junk does not spend a budget, and one kind's reports do not spend the +// other's. +func TestJunkSpendsNoBudget(t *testing.T) { + s, _, cfg := newService(t) + for range 3 * int(burst) { + if w := post(s, bytes.Repeat([]byte{9}, 400), ohttp.RequestType); w.Code != http.StatusBadRequest { + t.Fatalf("junk: %d", w.Code) + } + } + if got := send(t, s, cfg, inner(good).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("a real report after junk: %d", got) + } for i := range int(burst) { - if w := post(s, sealed, ohttp.RequestType); w.Code == http.StatusTooManyRequests { - t.Fatalf("limited after %d", i) + if got := send(t, s, cfg, inner(good).Marshal(reportwire.RequestSize)); got != http.StatusNoContent && i < int(burst)-1 { + t.Fatalf("energy %d: %d", i, got) } } - if w := post(s, sealed, ohttp.RequestType); w.Code != http.StatusTooManyRequests { - t.Fatalf("over budget: %d", w.Code) + if got := send(t, s, cfg, inner(good).Marshal(reportwire.RequestSize)); got != http.StatusTooManyRequests { + t.Fatalf("energy over budget: %d", got) + } + if got := send(t, s, cfg, inner(fleet).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("fleet beside a spent energy budget: %d", got) + } +} + +// A sender that stops halfway through its body is cut off. +func TestASlowBodyIsCutOff(t *testing.T) { + s, _, _ := newService(t) + old := bodyDeadline + bodyDeadline = 200 * time.Millisecond + t.Cleanup(func() { bodyDeadline = old }) + srv := httptest.NewServer(s) + defer srv.Close() + conn, err := net.Dial("tcp", srv.Listener.Addr().String()) + if err != nil { + t.Fatal(err) } - now = now.Add(time.Second) - defer func() { now = now.Add(-time.Second) }() - if w := post(s, sealed, ohttp.RequestType); w.Code != http.StatusOK { - t.Fatalf("after a second: %d", w.Code) + defer conn.Close() + _, _ = fmt.Fprintf(conn, "POST /v1/ohttp HTTP/1.1\r\nHost: x\r\nContent-Type: message/ohttp-req\r\nContent-Length: 1000\r\n\r\nabc") + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil || resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("slow body: %v %v", resp, err) } } diff --git a/internal/registry/energy.go b/internal/registry/energy.go index 9b92e07..c1730a9 100644 --- a/internal/registry/energy.go +++ b/internal/registry/energy.go @@ -1,6 +1,7 @@ package registry import ( + "database/sql" "time" "github.com/srcfl/ftw/go/protocol/energywire" @@ -34,6 +35,15 @@ func (r *Registry) EnergyInit() error { // EnergyAdd adds one report to its date's totals in one statement. False // means the date already holds energywire.MaxReports. func (r *Registry) EnergyAdd(rep energywire.Report) (bool, error) { + return energyAdd(r.db, rep) +} + +// execer is a database or a transaction. +type execer interface { + Exec(query string, args ...any) (sql.Result, error) +} + +func energyAdd(db execer, rep energywire.Report) (bool, error) { s := rep.Sums() var had [6]int for i, v := range s { @@ -41,7 +51,7 @@ func (r *Registry) EnergyAdd(rep energywire.Report) (bool, error) { had[i] = 1 } } - res, err := r.db.Exec(`INSERT INTO energy_days VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + res, err := db.Exec(`INSERT INTO energy_days VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(date) DO UPDATE SET reports = reports + 1, pv_generation_kwh = pv_generation_kwh + excluded.pv_generation_kwh, grid_import_kwh = grid_import_kwh + excluded.grid_import_kwh, diff --git a/internal/registry/fleet.go b/internal/registry/fleet.go index 97b6c67..8b64cec 100644 --- a/internal/registry/fleet.go +++ b/internal/registry/fleet.go @@ -22,10 +22,17 @@ func (r *Registry) FleetPut(report fleetwire.Report, now time.Time) (bool, error return false, err } defer tx.Rollback() + if ok, err := fleetPutTx(tx, report, now); err != nil || !ok { + return false, err + } + return true, tx.Commit() +} + +func fleetPutTx(tx *sql.Tx, report fleetwire.Report, now time.Time) (bool, error) { date := now.UTC().Format(time.DateOnly) day := fleetwire.NewDay(date) var old []byte - err = tx.QueryRow(`SELECT totals FROM fleet_days WHERE date = ?`, date).Scan(&old) + err := tx.QueryRow(`SELECT totals FROM fleet_days WHERE date = ?`, date).Scan(&old) if err != nil && !errors.Is(err, sql.ErrNoRows) { return false, err } @@ -47,7 +54,7 @@ func (r *Registry) FleetPut(report fleetwire.Report, now time.Time) (bool, error if _, err := tx.Exec(`DELETE FROM fleet_days WHERE date < ? OR date NOT IN (SELECT date FROM fleet_days ORDER BY date DESC LIMIT ?)`, now.UTC().AddDate(0, 0, 1-fleetwire.MaxDays).Format(time.DateOnly), fleetwire.MaxDays); err != nil { return false, err } - return true, tx.Commit() + return true, nil } func (r *Registry) FleetView(now time.Time) (fleetwire.View, error) { diff --git a/internal/registry/reports.go b/internal/registry/reports.go new file mode 100644 index 0000000..979a35e --- /dev/null +++ b/internal/registry/reports.go @@ -0,0 +1,66 @@ +package registry + +import ( + "crypto/sha256" + "database/sql" + "time" + + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" +) + +// ReportIDDays is how long a report's id is kept: the day it was sent and +// the next, when the gateway still takes it, and one more. +const ReportIDDays = 3 + +// ReportsInit creates the ids of the reports the gateway counted: a hash of +// each report's random id under the day the box sent it, so a replay or a +// retry counts once (RFC 9458, section 6.5). The table has no time, row id +// or arrival order: it is keyed by the hash. +func (r *Registry) ReportsInit() error { + _, err := r.db.Exec(`CREATE TABLE IF NOT EXISTS report_ids ( + day TEXT NOT NULL, + id BLOB NOT NULL, + PRIMARY KEY (day, id)) WITHOUT ROWID`) + return err +} + +// FleetPutOnce adds a fleet report to today's counters unless its id was +// counted already. +func (r *Registry) FleetPutOnce(day string, id []byte, report fleetwire.Report, now time.Time) (added, repeat bool, err error) { + return r.once(day, id, func(tx *sql.Tx) (bool, error) { return fleetPutTx(tx, report, now) }) +} + +// EnergyAddOnce adds an energy report to its date's totals unless its id was +// counted already. +func (r *Registry) EnergyAddOnce(day string, id []byte, rep energywire.Report) (added, repeat bool, err error) { + return r.once(day, id, func(tx *sql.Tx) (bool, error) { return energyAdd(tx, rep) }) +} + +// once records the id and runs add in one transaction. A repeat runs nothing. +// When add refuses (a full day), the id is not kept either. +func (r *Registry) once(day string, id []byte, add func(*sql.Tx) (bool, error)) (added, repeat bool, err error) { + sum := sha256.Sum256(id) + tx, err := r.db.Begin() + if err != nil { + return false, false, err + } + defer tx.Rollback() + res, err := tx.Exec(`INSERT INTO report_ids(day, id) VALUES (?, ?) ON CONFLICT DO NOTHING`, day, sum[:16]) + if err != nil { + return false, false, err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err == nil, err + } + if added, err = add(tx); err != nil || !added { + return false, false, err + } + return true, false, tx.Commit() +} + +// ReportsPrune drops ids older than ReportIDDays. +func (r *Registry) ReportsPrune(now time.Time) error { + _, err := r.db.Exec(`DELETE FROM report_ids WHERE day < ?`, now.UTC().AddDate(0, 0, 1-ReportIDDays).Format(time.DateOnly)) + return err +} diff --git a/internal/server/reports_e2e_test.go b/internal/server/reports_e2e_test.go index cab0e97..20ed7d5 100644 --- a/internal/server/reports_e2e_test.go +++ b/internal/server/reports_e2e_test.go @@ -3,6 +3,7 @@ package server import ( "bytes" "crypto/ecdh" + "crypto/rand" "encoding/json" "io" "net/http" @@ -72,7 +73,9 @@ func TestBothReportsReachTheirTotalsThroughARelay(t *testing.T) { fleetBody, _ := json.Marshal(fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.146.0", Channel: "stable", Drivers: []string{"sungrow"}, Battery: "5-15", Zone: "SE4", Age: "1-6m"}) // Three energy reports and two fleet reports, each alone. for i, body := range [][]byte{energyBody, fleetBody, energyBody, fleetBody, energyBody} { - inner, ok := reportwire.Request(body) + id := make([]byte, reportwire.IDBytes) + _, _ = rand.Read(id) + inner, ok := reportwire.Request(reportwire.Report{ID: id, Day: time.Now().UTC().Format(time.DateOnly), Body: body}) if !ok { t.Fatal("report does not fit") } From a54b2ebbea050faa2c1d14c45f3ee6d6e9bac096 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:28:55 +0200 Subject: [PATCH 5/8] docs: the floor of five counts reports, accepted; tokens later Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3fffd73..94db4c5 100644 --- a/README.md +++ b/README.md @@ -567,12 +567,13 @@ homes. report, the relay above all, cannot poll the view and see which count it moved. -Limits, plainly. A report cannot prove that a real box sent it. Anyone with -the public key configuration can post reports, and the relay can drop some. -One real report and four all-zero fakes would publish that home's six sums -under "5 reports". The floor of five counts reports, not boxes. This is an -open design question for Fredrik; bounds and caps do not solve it. -Privacy Pass-style tokens would; they are not built. The +Limits, plainly. The floor of five counts reports, not boxes; Fredrik +accepted this on 24 October 2026. A report cannot prove that a real box sent +it. The relay alone can drop the other reports and add fakes to isolate one +home's day, and that shows on our side as missing reports. Anyone else can +do it only on a day when one real home reported. Rejecting replays and +listing closed days only remove the cheap attacks. Anonymous tokens (Privacy +Pass) are the later fix, once the fleet grows or someone shows a need. The relay and Sourceful together could join reports to addresses by time. This service also holds readings from homes with the readable copy on, including 48 hours of history tiles, from which the same daily sums could be computed; From 2a7c3a60b218dc7ab1f2d9adb04ce3d9ab6f4b5c Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 10:10:58 +0200 Subject: [PATCH 6/8] Answer a repeat before the budget, refuse any partial key block The gateway now looks a report id up, read-only, before it spends the kind's budget. A repeat gets a sealed 204 and costs nothing, so one captured request sent many times cannot crowd out real reports. The OHTTP key file must hold whole PKCS #8 blocks and nothing else. A block cut short at either end, or text between blocks, stops the start. Co-Authored-By: Claude Opus 5.5 --- cmd/ftwcloudd/ohttp.go | 24 ++++++++++++------------ cmd/ftwcloudd/ohttp_test.go | 18 ++++++++++++------ internal/gateway/gateway.go | 9 ++++++++- internal/gateway/gateway_test.go | 18 ++++++++++++++++++ internal/registry/reports.go | 13 +++++++++++++ 5 files changed, 63 insertions(+), 19 deletions(-) diff --git a/cmd/ftwcloudd/ohttp.go b/cmd/ftwcloudd/ohttp.go index 43718ed..06363cc 100644 --- a/cmd/ftwcloudd/ohttp.go +++ b/cmd/ftwcloudd/ohttp.go @@ -32,16 +32,21 @@ func loadOHTTPKeys(path string) (*ohttp.Gateway, error) { if err != nil { return nil, fmt.Errorf("ohttp key: %w", err) } + // Each key is one whole PEM block, with nothing but blank space around + // it. Anything else, such as a block cut short in a rotation, is refused + // wherever it sits: pem.Decode would skip it and load the rest. + const begin, end = "-----BEGIN PRIVATE KEY-----", "-----END PRIVATE KEY-----" var keys []ohttp.Key - for { - var block *pem.Block - block, raw = pem.Decode(raw) - if block == nil { - break + for rest := bytes.TrimSpace(raw); len(rest) > 0; rest = bytes.TrimSpace(rest) { + stop := bytes.Index(rest, []byte(end)) + if !bytes.HasPrefix(rest, []byte(begin)) || stop < 0 || bytes.Count(rest[:stop], []byte("-----BEGIN")) != 1 { + return nil, fmt.Errorf("ohttp key: %s holds data that is not a whole PKCS #8 key", path) } - if block.Type != "PRIVATE KEY" { - return nil, fmt.Errorf("ohttp key: %s holds a %s, not a PKCS #8 private key", path, block.Type) + block, after := pem.Decode(rest[:stop+len(end)]) + if block == nil || len(bytes.TrimSpace(after)) > 0 { + return nil, fmt.Errorf("ohttp key: %s holds data that is not a whole PKCS #8 key", path) } + rest = rest[stop+len(end):] parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) if err != nil { return nil, fmt.Errorf("ohttp key: %w", err) @@ -52,11 +57,6 @@ func loadOHTTPKeys(path string) (*ohttp.Gateway, error) { } keys = append(keys, ohttp.Key{ID: ohttpKeyID(x.PublicKey().Bytes()), Private: x}) } - // A block cut short, as a rotation pasted in part would leave it, must - // not leave the new key quietly unloaded. - if len(bytes.TrimSpace(raw)) > 0 { - return nil, fmt.Errorf("ohttp key: %s has data after its last whole key", path) - } if len(keys) == 0 { return nil, fmt.Errorf("ohttp key: %s holds no key", path) } diff --git a/cmd/ftwcloudd/ohttp_test.go b/cmd/ftwcloudd/ohttp_test.go index b20c3d2..81662f9 100644 --- a/cmd/ftwcloudd/ohttp_test.go +++ b/cmd/ftwcloudd/ohttp_test.go @@ -59,12 +59,18 @@ func TestAGatewayKeyLoadsAsThePinSays(t *testing.T) { // A second block cut short, as a rotation pasted in part leaves it, is // refused rather than quietly left unloaded. - cut := filepath.Join(dir, "cut.key") - if err := os.WriteFile(cut, append(a, b[:len(b)/2]...), 0o600); err != nil { - t.Fatal(err) - } - if _, err := loadOHTTPKeys(cut); err == nil || !strings.Contains(err.Error(), "after its last whole key") { - t.Fatalf("a cut second key: %v", err) + for name, data := range map[string][]byte{ + "cut after a whole key": append(append([]byte{}, a...), b[:len(b)/2]...), + "cut before a whole key": append(append([]byte{}, b[:len(b)/2]...), a...), + "text between keys": append(append(append([]byte{}, a...), []byte("note\n")...), b...), + } { + cut := filepath.Join(dir, "cut.key") + if err := os.WriteFile(cut, data, 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadOHTTPKeys(cut); err == nil || !strings.Contains(err.Error(), "not a whole PKCS #8 key") { + t.Fatalf("%s: %v", name, err) + } } } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index e911a8e..cdc50b0 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -135,13 +135,20 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { // add reads the one request the gateway answers and adds its report to the // day totals of its kind, once: a report whose id was counted already is -// answered 204 and adds nothing, so a replay or a retry is harmless. +// answered 204, adds nothing and spends no budget, so a replay or a retry is +// harmless and cannot crowd out real reports. func (s *Service) add(msg []byte) int { now := s.now() report, ok := reportwire.Read(msg) if !ok || !reportwire.Fresh(report.Day, now) { return http.StatusBadRequest } + switch seen, err := s.reg.ReportSeen(report.Day, report.ID); { + case err != nil: + return http.StatusServiceUnavailable + case seen: + return http.StatusNoContent + } var added, repeat bool var err error switch reportwire.Schema(report.Body) { diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index b25f54c..e7ff290 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -223,6 +223,24 @@ func TestAReportCountsOnceHoweverOftenItArrives(t *testing.T) { } } +// A repeat is answered before any budget is spent, so one captured request +// sent many times cannot crowd out the real reports. +func TestARepeatSpendsNoBudget(t *testing.T) { + s, reg, cfg := newService(t) + sealed, _, _ := ohttp.Encapsulate(cfg, inner(fleet).Marshal(reportwire.RequestSize)) + for i := range 2 * int(burst) { + if w := post(s, sealed, ohttp.RequestType); w.Code != http.StatusOK { + t.Fatalf("repeat %d: %d", i, w.Code) + } + } + if got := send(t, s, cfg, inner(fleet).Marshal(reportwire.RequestSize)); got != http.StatusNoContent { + t.Fatalf("a new report after repeats: %d", got) + } + if view, _ := reg.FleetView(now); len(view.Days) != 1 || view.Days[0].Reports != 2 { + t.Fatalf("fleet %+v", view) + } +} + // A request sent before yesterday is refused, so a captured one cannot be // posted again once its id is forgotten. Yesterday's still counts. func TestAnOldRequestIsRefused(t *testing.T) { diff --git a/internal/registry/reports.go b/internal/registry/reports.go index 979a35e..1ce7ed6 100644 --- a/internal/registry/reports.go +++ b/internal/registry/reports.go @@ -3,6 +3,7 @@ package registry import ( "crypto/sha256" "database/sql" + "errors" "time" "github.com/srcfl/ftw/go/protocol/energywire" @@ -59,6 +60,18 @@ func (r *Registry) once(day string, id []byte, add func(*sql.Tx) (bool, error)) return true, false, tx.Commit() } +// ReportSeen reports whether a report's id was counted already, without +// recording it, so a repeat can be answered before any budget is spent. +func (r *Registry) ReportSeen(day string, id []byte) (bool, error) { + sum := sha256.Sum256(id) + var one int + err := r.rd.QueryRow(`SELECT 1 FROM report_ids WHERE day = ? AND id = ?`, day, sum[:16]).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + // ReportsPrune drops ids older than ReportIDDays. func (r *Registry) ReportsPrune(now time.Time) error { _, err := r.db.Exec(`DELETE FROM report_ids WHERE day < ?`, now.UTC().AddDate(0, 0, 1-ReportIDDays).Format(time.DateOnly)) From 8219d20be8c8c0e0e02313f30b70c4b8ccf10ce8 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 08:19:35 +0200 Subject: [PATCH 7/8] feat: keep the service's hours for 90 days and serve them to the stats Worker ftwcloudd now keeps an hourly summary of every series it counts in the registry (ops_hours), for 90 days: each counter's rise, each gauge's low, mean and high, each latency's buckets, how much of the hour a process ran and how many started. A deploy hands over its part of the hour after the drain; the next process adds the rest. GET /v1/ops serves a week of settled hours at a time, with the exact totals of the anonymous fleet and energy reports for closed days, only to the key the project stats Worker holds. The service holds the key's SHA-256 (-ops-key-sha256); without it the route answers 404 to everyone. New series: sockets by transport (QUIC or WebSocket), bytes written to boxes and phones, MCP requests and tool calls by outcome and their time, web push answers, and the drain's sockets. Every label is a fixed word (TestCountsNameNobody), and traffic makes no new series and leaves no home, device, key or address in an hour (TestAnHourOfCountsNamesNobody). Stacked on #50, whose energy day totals it reads. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 +- README.md | 52 ++++- cmd/ftwcloudd/main.go | 24 ++- cmd/ftwcloudd/main_test.go | 33 +++ deploy/README.md | 16 ++ deploy/compose.yml | 3 + internal/appsession/appsession.go | 12 +- internal/mcp/mcp.go | 26 +++ internal/mcp/mcp_test.go | 15 ++ internal/metrics/hours.go | 207 ++++++++++++++++++ internal/metrics/hours_test.go | 129 ++++++++++++ internal/metrics/metrics.go | 23 +- internal/ops/ops.go | 278 +++++++++++++++++++++++++ internal/ops/ops_test.go | 335 ++++++++++++++++++++++++++++++ internal/registry/ops.go | 68 ++++++ internal/server/ops_e2e_test.go | 95 +++++++++ internal/server/server.go | 16 +- internal/server/server_test.go | 5 + internal/uplink/uplink.go | 10 +- internal/webpush/webpush.go | 15 ++ internal/wire/wire.go | 10 + privacy/internal-operations.md | 83 ++++++++ 22 files changed, 1451 insertions(+), 12 deletions(-) create mode 100644 internal/metrics/hours.go create mode 100644 internal/metrics/hours_test.go create mode 100644 internal/ops/ops.go create mode 100644 internal/ops/ops_test.go create mode 100644 internal/registry/ops.go create mode 100644 internal/server/ops_e2e_test.go create mode 100644 privacy/internal-operations.md diff --git a/CLAUDE.md b/CLAUDE.md index 184a269..59e620c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,8 +42,8 @@ maintains it. Memory holds live readings, and only while the roster's grant serves a copy; SQLite holds only the home id, the box key, the signed roster, owners' suspensions, tombstones, APNs registrations, the hashes of - extension tokens when each away box's alert is due, browser notice sent markers and - anonymous daily fleet and energy totals. Browser endpoints and box-encrypted offline + extension tokens when each away box's alert is due, browser notice sent markers, + anonymous daily fleet and energy totals, and the service's own hourly counts. Browser endpoints and box-encrypted offline notices live only in the signed roster. Nothing logs addresses. - **A phone can only narrow, or talk to its own box.** On `/v1/app` it can @@ -70,6 +70,10 @@ maintains it. one, because other parties' clients follow no redirect. - **Counts name nobody.** `/metrics` listens on loopback only, and it and the minute's log line sum over all homes: no home id, device or address. + The registry keeps the same sums an hour at a time for 90 days, and + `/v1/ops` serves them, with the closed fleet and energy day totals, only to + the stats Worker's key. A label is a word fixed in the code + (`TestCountsNameNobody`); never add one that could hold a home. - **A deploy drains.** On SIGTERM every socket, handshakes included, hears 4503 with a spread before the process exits, and no drained box starts `box.unreachable`. A part that holds sockets is a `drainer` in diff --git a/README.md b/README.md index 94db4c5..bda3b0b 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ to it yet. - What the service holds follows the newest roster's grant, never `hello`: the readable copy can turn on or off while the uplink stays up. - The registry (SQLite) holds the home id, the box key, the signed roster, - owners' suspensions, APNs registrations, the hashes of extension tokens - and when an away box's alert is due. A box whose owner turns the readable + owners' suspensions, APNs registrations, the hashes of extension tokens, + when an away box's alert is due, and the service's own hourly counts. A box whose owner turns the readable copy off gets that copy deleted and a signed receipt (`purged {what: "copy"}`); a box whose link ends gets its home deleted and a receipt. - **Room to grow.** Each home lives on one shard; an instance that does not @@ -36,7 +36,8 @@ to it yet. one instance holds every home. A deploy closes every socket with 4503 and a spread, so boxes and phones come back over seconds rather than at once. `/metrics` on loopback and one log line a minute count what the service - does without naming a home. `deploy/README.md` has the load numbers and + does without naming a home; the registry keeps the same counts an hour + at a time for 90 days ([Internal operations](#internal-operations)). `deploy/README.md` has the load numbers and when to add a shard. - **A restart forgets no one.** At start the service reads every signed roster back from the registry and checks it with the home's key, so a @@ -585,3 +586,48 @@ OHTTP should be outside GDPR, but reading data from the owner's equipment to make them may still need consent under the ePrivacy rule (Swedish LEK, chapter 9, section 28). If it does, neither the opt-out for boxes without Cloud nor the requirement with Cloud holds. + +## Internal operations + +Sourceful runs the service and the fleet from one internal view: the private +page of stats.ftw.energy, behind Cloudflare Access. It needs no system of +its own. The service keeps its counts; the stats Worker pulls them once an +hour into its D1 database. + +**What the service keeps.** Every hour `internal/metrics` hands the hour to +the registry (`ops_hours`, one row an hour): how much each counter rose, +each gauge's low, mean and high over the minutes sampled, each latency's +buckets, how much of the hour a process ran, and how many processes started +in it. A deploy hands over its part of the hour after the drain, and the +next process adds the rest to the same row. The series are the ones +`/metrics` serves: sockets by kind and by transport (QUIC or WebSocket), +homes held and connected, bytes written to boxes and phones, tun frames and +bytes, drops by reason, push queues and the answers of Apple and of browsers' +push services, MCP requests and tool calls by outcome and their time, +handshake and forward latency, the drain's sockets and the Go runtime. +Hours go after 90 days, at each write and at upkeep. + +**What it serves.** `GET /v1/ops` answers only the key the Worker holds, +`Authorization: Bearer `. The service holds the key's SHA-256 +(`-ops-key-sha256`), which is no secret; without it the route answers 404 +to everyone, and the hours are kept all the same. An answer carries at most +a week of settled hours (`?since=`, five minutes after an hour ends), and the +exact totals of the anonymous fleet and energy reports for closed days: a +fleet day once it has ended, an energy date once no report may name it, as +`/v1/energy/public` has it. Below the public group size of 5 too, but never +an open day, so two answers never differ by one report. `/v1/ops` is +`ftw.cloud-ops/1`; the Worker refuses any field it does not name. + +**Why none of it is about a household.** Each series sums over the whole +process. A label is a word fixed in the code, never a home, a device, a key +or an address (`TestCountsNameNobody`), and no series is made for a home +(`TestAnHourOfCountsNamesNobody`). The fleet and energy days are totals of +reports that carry no id and keep nothing per report. The limits, plainly: +while the fleet is a handful of homes, an hour's phone count or a closed +day's single report says something about those few, and staff who know +them could guess. The feed gives nobody more than the totals, and nobody +outside Access sees them. [privacy/internal-operations.md](privacy/internal-operations.md) +has the argument and the retention in full. + +Per-home data stays where it is: in memory for the home's own phones, and in +support access only when an owner opts in for a ticket. diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 6a0e1ca..f9c7690 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -44,6 +44,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/mcp" "github.com/srcfl/ftw-cloud/internal/metrics" "github.com/srcfl/ftw-cloud/internal/oauth" + "github.com/srcfl/ftw-cloud/internal/ops" "github.com/srcfl/ftw-cloud/internal/push" "github.com/srcfl/ftw-cloud/internal/quicdoor" "github.com/srcfl/ftw-cloud/internal/registry" @@ -104,6 +105,7 @@ func serve(args []string) error { shardsPath := fs.String("shards", "", "shard directory, JSON; without it this instance holds every home") statsPolicy := fs.String("statistics-policy", "", "approved statistics policy; empty keeps hourly collection off pending C8 sign-off") metricsAddr := fs.String("metrics-listen", defaultMetricsAddr, "loopback address for /metrics; empty turns it off") + opsKey := fs.String("ops-key-sha256", "", "hex SHA-256 of the key the project stats Worker reads /v1/ops with; empty serves no one") restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial") restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts") h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc") @@ -179,9 +181,14 @@ func serve(args []string) error { if err != nil { return err } + // The service's own hours, kept 90 days for Sourceful's internal view. + opsSvc, err := ops.New(reg, *opsKey, shards.Self()) + if err != nil { + return err + } tokens := token.New(st, reg, log) tokens.Shards = shards - srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Gateway: gatewaySvc, Statistics: usageSvc, Log: log} + srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Gateway: gatewaySvc, Statistics: usageSvc, Ops: opsSvc, Log: log} // Assistants: the authorization server, with agent keys in ftwkeysd. agentKeys := keys.Dial(*keysSocket) @@ -348,6 +355,10 @@ func serve(args []string) error { } } go metrics.Default.Summarise(ctx, os.Stdout, shards.Self(), time.Minute) + // The same series, an hour at a time, into the registry. A deploy hands + // over the hour so far after the drain, so the drain is in it. + keeper := metrics.Default.NewKeeper(opsSvc, func(err error) { log.Warn("an hour of metrics was not kept", "err", err) }) + go keeper.Run(ctx) // A deploy: stop accepting, tell every box and phone to come back within // a spread the next instance can take, let the queued alerts go, then @@ -382,9 +393,13 @@ func serve(args []string) error { left += d.WaitClosed(time.Until(deadline)) } log.Info("drained for a restart", "sockets", sockets, "still_open", left, "spread", spread) + drains.Inc() + drainedSockets.Add(int64(sockets)) + drainLeftOpen.Add(int64(left)) if left := pushSvc.Flush(3 * time.Second); left > 0 { log.Warn("alerts queued or waiting to go again at exit; they are lost", "count", left) } + keeper.Close() stopPush() }() if door != nil { @@ -425,6 +440,13 @@ type drainer interface { // drainWait bounds the wait for the clients' answers to the drain's close. const drainWait = 2 * time.Second +// What a deploy's drain did, kept in the hour it ends. +var ( + drains = metrics.Default.Counter("ftwcloud_drains_total", "Drains for a restart.") + drainedSockets = metrics.Default.Counter("ftwcloud_drained_sockets_total", "Sockets a drain closed with 4503, handshakes included.") + drainLeftOpen = metrics.Default.Counter("ftwcloud_drain_left_open_total", "Sockets whose clients had not answered the drain's close when it stopped waiting.") +) + // defaultRestartRate is how many sockets a second one instance takes on // the host, with room to spare: deploy/README.md, "Load", has the numbers. const defaultRestartRate = 500 diff --git a/cmd/ftwcloudd/main_test.go b/cmd/ftwcloudd/main_test.go index 2694c31..79736e9 100644 --- a/cmd/ftwcloudd/main_test.go +++ b/cmd/ftwcloudd/main_test.go @@ -1,10 +1,43 @@ package main import ( + "regexp" + "slices" + "strings" "testing" "time" + + "github.com/srcfl/ftw-cloud/internal/metrics" ) +// Every series the service keeps, which /v1/ops keeps for 90 days, carries +// only labels from this list, each a word fixed in the code. A label that +// could hold a home, a device or an address is a change to argue for here. +func TestCountsNameNobody(t *testing.T) { + var b strings.Builder + if _, err := metrics.Default.WriteTo(&b); err != nil { + t.Fatal(err) + } + pair := regexp.MustCompile(`([a-z_]+)="([^"]*)"`) + word := regexp.MustCompile(`^[a-z0-9_]{1,40}$|^[0-9.e+-]+$|^\+Inf$`) + allowed := []string{"kind", "reason", "path", "dir", "result", "transport", "outcome", "le"} + n := 0 + for _, line := range strings.Split(b.String(), "\n") { + if strings.HasPrefix(line, "#") || line == "" { + continue + } + n++ + for _, m := range pair.FindAllStringSubmatch(line, -1) { + if !slices.Contains(allowed, m[1]) || !word.MatchString(m[2]) { + t.Fatalf("label %s=%q in %s", m[1], m[2], line) + } + } + } + if n < 50 { + t.Fatalf("only %d series: is every package's set registered?", n) + } +} + func TestTheRestartSpreadFollowsTheSockets(t *testing.T) { for _, c := range []struct { sockets, rate int diff --git a/deploy/README.md b/deploy/README.md index b36ad15..f8039be 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -215,6 +215,22 @@ closed port, so the spread's floor can drop to a few seconds. Logs (the `awslogs` log driver, and `logs:CreateLogStream` and `logs:PutLogEvents` for the instance role, which the owner adds), each field is a metric in the `FTWCloud` namespace, by shard. +- **The hours, for 90 days.** The registry keeps the same counts an hour + at a time, and the private page of stats.ftw.energy reads them from + `/v1/ops` once an hour ([README](../README.md#internal-operations)). To + turn the feed on, make a key and give it to both ends: + + ```bash + key=$(openssl rand -base64 32 | tr '+/' '-_' | tr -d '=') + printf %s "$key" | npx wrangler secret put CLOUD_OPS_KEY --config stats/wrangler.jsonc # in srcfl/ftw-webapp + printf %s "$key" | shasum -a 256 | cut -d' ' -f1 # FTW_CLOUD_OPS_KEY_SHA256 + ``` + + Put the hash in `/etc/ftw-cloud/compose.env` as + `FTW_CLOUD_OPS_KEY_SHA256=`; the next deploy serves the feed. The hash is + no secret, the key is: it lives only in the Worker. Until the service has + a hash the Worker logs `awaiting key on Cloud`; while the two differ, a + 401. To rotate, do all three again. - What to watch: `forward_seconds_*_p99` (time inside the service), `drops_*` (any rise), `push_queued_alert`, `homes_connected`, `go_memory_bytes`. diff --git a/deploy/compose.yml b/deploy/compose.yml index cf23143..cef7cc0 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -30,6 +30,9 @@ services: - -noise-key=/run/secrets/noise/noise.key - -ohttp-key=/run/secrets/ohttp/ohttp.key - -keys-socket=/run/ftw-cloud-keys/keys.sock + # The SHA-256 of the key the stats Worker reads /v1/ops with: no + # secret. Empty turns the feed off; the hours are kept all the same. + - -ops-key-sha256=${FTW_CLOUD_OPS_KEY_SHA256:-} environment: # /source publishes it, so anyone can compare it with what CI built. FTW_CLOUD_IMAGE: ${FTW_CLOUD_IMAGE} diff --git a/internal/appsession/appsession.go b/internal/appsession/appsession.go index 75f122e..22678ba 100644 --- a/internal/appsession/appsession.go +++ b/internal/appsession/appsession.go @@ -94,6 +94,8 @@ type Exchanger interface { // What the app sessions count, summed over all homes. var ( appSockets = metrics.Sockets("app") + appBytesOut = metrics.BytesOut("app") + appOver = map[string]*metrics.Gauge{cloudwire.TransportQUIC: metrics.TransportSockets("app", cloudwire.TransportQUIC), cloudwire.TransportWebSocket: metrics.TransportSockets("app", cloudwire.TransportWebSocket)} appHandshake = metrics.Handshake("app") onceHandshake = metrics.Handshake("once") envelopeOut = metrics.Forward("envelope") @@ -221,6 +223,10 @@ func (s *Server) ServeConn(conn wire.Conn, home string) { defer conn.Close() appSockets.Add(1) defer appSockets.Add(-1) + if over := appOver[wire.Transport(conn)]; over != nil { + over.Add(1) + defer over.Add(-1) + } if !cloudwire.ValidHome(home) { _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(cloudwire.CloseProtocol, ""), time.Now().Add(time.Second)) return @@ -538,7 +544,11 @@ func (c *phoneConn) writeAs(kind int, msg []byte) error { if err := c.conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { return err } - return c.conn.WriteMessage(wire.Kind(c.conn, kind), msg) + if err := c.conn.WriteMessage(wire.Kind(c.conn, kind), msg); err != nil { + return err + } + appBytesOut.Add(int64(len(msg))) + return nil } // forward sends one box envelope as the box sent it. An envelope too large diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 71121e4..b2763c1 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -32,9 +32,22 @@ import ( "sync" "time" + "github.com/srcfl/ftw-cloud/internal/metrics" "github.com/srcfl/ftw-cloud/internal/oauth" ) +// What the service counts of assistants, summed over every home and agent. +var ( + mcpOutcomes = map[string]*metrics.Counter{} + mcpCall = metrics.Default.Histogram("ftwcloud_mcp_call_seconds", "From a tool's start to its result, the box's answer included.") +) + +func init() { + for _, o := range []string{"refused", "rate_limited", "ok", "tool_error", "failed", "unknown_tool", "unsupported", "step_up", "busy"} { + mcpOutcomes[o] = metrics.Default.Counter("ftwcloud_mcp_total", "MCP requests refused before a call, and tool calls, by outcome.", "outcome", o) + } +} + // Versions. const ( // Modern is the revision this server is written to. @@ -196,10 +209,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { _ = http.NewResponseController(w).SetReadDeadline(time.Now().Add(10 * time.Second)) grant, aerr := s.auth.Authenticate(r) if aerr != nil { + mcpOutcomes["refused"].Inc() oauth.Challenge(w, profileAuthError(r, aerr)) return } if !s.limit.allow(grant.Subject.Home+"/"+grant.Subject.Agent, s.now()) { + mcpOutcomes["rate_limited"].Inc() w.Header().Set("Retry-After", "1") writeRPCError(w, http.StatusTooManyRequests, nil, codeInternal, "too many requests; slow down", nil) return @@ -375,6 +390,7 @@ type callParams struct { func (s *Server) call(w http.ResponseWriter, r *http.Request, id json.RawMessage, p callParams, g *oauth.Grant, modern bool) { if unsupportedDiagnostic(p.Name) { + mcpOutcomes["unsupported"].Inc() result := toolError("This connector does not provide this operation (E_UNSUPPORTED). Use measurement_diagnostics, health_get or logs_tail.") result["structuredContent"] = map[string]any{"source": "service", "data": map[string]any{"available": false, "code": "E_UNSUPPORTED", "operation": p.Name, "alternatives": []string{"measurement_diagnostics", "health_get", "logs_tail"}}} writeResult(w, id, modern, result) @@ -385,6 +401,7 @@ func (s *Server) call(w http.ResponseWriter, r *http.Request, id json.RawMessage t = nil } if t == nil { + mcpOutcomes["unknown_tool"].Inc() status := http.StatusOK if modern { status = http.StatusBadRequest @@ -395,6 +412,7 @@ func (s *Server) call(w http.ResponseWriter, r *http.Request, id json.RawMessage required := t.requiredScopes() if slices.ContainsFunc(required, func(scope string) bool { return !g.Has(scope) }) { // Step up: the client asks the owner for the right, then retries. + mcpOutcomes["step_up"].Inc() scopes := append(slices.Clone(g.Scopes), required...) slices.Sort(scopes) scopes = slices.Compact(scopes) @@ -410,6 +428,7 @@ func (s *Server) call(w http.ResponseWriter, r *http.Request, id json.RawMessage case s.inflight <- struct{}{}: defer func() { <-s.inflight }() default: + mcpOutcomes["busy"].Inc() writeResult(w, id, modern, toolError("FTW Cloud is busy. Try again in a moment.")) return } @@ -418,22 +437,29 @@ func (s *Server) call(w http.ResponseWriter, r *http.Request, id json.RawMessage args = json.RawMessage("{}") } ctx := r.Context() + start := time.Now() data, err := t.run(ctx, &callCtx{s: s, grant: g, rpcID: string(id)}, args) + mcpCall.Since(start) if err != nil { var ue userError + outcome := "tool_error" if !errors.As(err, &ue) { s.log.Warn("a tool call failed", "tool", t.name, "err", err) ue = userError("The call failed inside FTW Cloud.") + outcome = "failed" } + mcpOutcomes[outcome].Inc() writeResult(w, id, modern, toolError(string(ue))) return } if failure, ok := data.(failedToolData); ok { + mcpOutcomes["tool_error"].Inc() result := toolResult(g.Subject.Home, failure.data) result["isError"] = true writeResult(w, id, modern, result) return } + mcpOutcomes["ok"].Inc() writeResult(w, id, modern, toolResult(g.Subject.Home, data)) } diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go index e9a550a..a656c8f 100644 --- a/internal/mcp/mcp_test.go +++ b/internal/mcp/mcp_test.go @@ -712,3 +712,18 @@ func TestARestartDrainsAgentSessions(t *testing.T) { t.Fatalf("a drained instance opened a session: %d handshakes", hs) } } + +// Calls are counted by outcome and timed, summed over every home and agent. +func TestCallsAreCountedByOutcome(t *testing.T) { + w := newWorld(t) + b := w.connectBox() + a := w.connect(b, "") + ok, unknown, took := mcpOutcomes["ok"].Load(), mcpOutcomes["unknown_tool"].Load(), mcpCall.Count() + if _, isErr, text := a.call("status", nil); isErr { + t.Fatal(text) + } + _ = a.rpc("tools/call", map[string]any{"name": "no_such_tool"}, false) + if mcpOutcomes["ok"].Load() != ok+1 || mcpOutcomes["unknown_tool"].Load() != unknown+1 || mcpCall.Count() != took+1 { + t.Fatalf("ok %d, unknown %d, timed %d", mcpOutcomes["ok"].Load()-ok, mcpOutcomes["unknown_tool"].Load()-unknown, mcpCall.Count()-took) + } +} diff --git a/internal/metrics/hours.go b/internal/metrics/hours.go new file mode 100644 index 0000000..aacfcf0 --- /dev/null +++ b/internal/metrics/hours.go @@ -0,0 +1,207 @@ +package metrics + +import ( + "context" + "slices" + "sync" + "time" +) + +// Hour is one clock hour of the service, for the operator to keep: how much +// each counter rose, each gauge's low, high and mean over the minutes +// sampled, and each histogram's buckets. Like every series it sums over the +// whole process: no home, device or address. Keys are the names of the +// minute's log line (Summarise). +type Hour struct { + // Start is the hour, in Unix milliseconds, UTC. + Start int64 `json:"start_ms"` + // Ms is how much of the hour a process sampled. Less than an hour + // means the hour held a start, a stop or a crash. + Ms int64 `json:"ms"` + // Starts is how many processes started in the hour: deploys and crashes. + Starts int64 `json:"starts"` + Counters map[string]int64 `json:"counters"` + Gauges map[string]GaugeHour `json:"gauges"` + Histograms map[string][]int64 `json:"histograms"` +} + +// GaugeHour is a gauge over an hour: its low, its high, and the sum and +// number of its samples. +type GaugeHour struct { + Min int64 `json:"min"` + Max int64 `json:"max"` + Sum int64 `json:"sum"` + N int64 `json:"n"` +} + +func newHour(t time.Time) Hour { + return Hour{Start: t.UTC().Truncate(time.Hour).UnixMilli(), Counters: map[string]int64{}, + Gauges: map[string]GaugeHour{}, Histograms: map[string][]int64{}} +} + +// Merge adds o, a later part of the same hour, as when a process restarts +// inside it. +func (h *Hour) Merge(o Hour) { + h.Ms += o.Ms + h.Starts += o.Starts + for k, v := range o.Counters { + h.Counters[k] += v + } + for k, g := range o.Gauges { + h.Gauges[k] = h.Gauges[k].merge(g) + } + for k, b := range o.Histograms { + old := h.Histograms[k] + if len(old) != len(b) { + // The buckets changed between versions: the later ones win. + h.Histograms[k] = slices.Clone(b) + continue + } + for i := range b { + old[i] += b[i] + } + } +} + +func (g GaugeHour) merge(o GaugeHour) GaugeHour { + if g.N == 0 { + return o + } + if o.N == 0 { + return g + } + return GaugeHour{Min: min(g.Min, o.Min), Max: max(g.Max, o.Max), Sum: g.Sum + o.Sum, N: g.N + o.N} +} + +func (g GaugeHour) add(v int64) GaugeHour { return g.merge(GaugeHour{Min: v, Max: v, Sum: v, N: 1}) } + +// Quantile is the upper bound, in seconds, of the LatencyBounds bucket that +// holds the q-th quantile of counts, +Inf past the last bound, or 0 without +// any. +func Quantile(counts []int64, q float64) float64 { return quantile(LatencyBounds, counts, q) } + +// HourStore keeps hours. PutHour merges h into the hour already kept for +// h.Start, if there is one. +type HourStore interface { + PutHour(h Hour) error +} + +// Keeper rolls the series up into hours and hands each to a store when it +// ends. Run samples it on each whole minute; Close hands over the hour it is +// in, which a deploy calls after the drain so the drain's counts are in it. +type Keeper struct { + set *Set + store HourStore + failed func(error) + now func() time.Time + + mu sync.Mutex + last state + at time.Time + hour Hour + closed bool +} + +// NewKeeper starts the hour now, counting this process's start in it. +// failed hears each hour the store would not take, which is then lost. +func (s *Set) NewKeeper(store HourStore, failed func(error)) *Keeper { + k := &Keeper{set: s, store: store, failed: failed, now: time.Now} + k.begin() + return k +} + +func (k *Keeper) begin() { + k.at = k.now() + k.last = k.set.state() + k.hour = newHour(k.at) + k.hour.Starts = 1 + k.gauges(k.last) +} + +// Run samples on each whole minute until ctx ends. The minute stays the +// unit after a slow tick, so no interval crosses an hour's end. +func (k *Keeper) Run(ctx context.Context) { + for { + now := k.now() + t := time.NewTimer(now.Truncate(time.Minute).Add(time.Minute).Sub(now)) + select { + case <-ctx.Done(): + t.Stop() + return + case <-t.C: + k.sample() + } + } +} + +// sample adds the time since the last sample to the hour it began in, and +// hands that hour over once the clock has left it. +func (k *Keeper) sample() { + k.mu.Lock() + defer k.mu.Unlock() + if k.closed { + return + } + now, cur := k.now(), k.set.state() + k.rise(cur, now) + if start := now.UTC().Truncate(time.Hour).UnixMilli(); start != k.hour.Start { + k.put(k.hour) + k.hour = newHour(now) + } + k.gauges(cur) +} + +// Close hands over the hour so far and stops the keeper. +func (k *Keeper) Close() { + k.mu.Lock() + defer k.mu.Unlock() + if k.closed { + return + } + k.closed = true + k.rise(k.set.state(), k.now()) + k.put(k.hour) +} + +// rise adds each counter's and histogram's rise since the last sample. +func (k *Keeper) rise(cur state, now time.Time) { + k.hour.Ms += max(0, now.Sub(k.at).Milliseconds()) + for x, v := range cur { + before := k.last[x] + switch x.kind { + case counter: + var prev int64 + if len(before) == 1 { + prev = before[0] + } + k.hour.Counters[field(x)] += v[0] - prev + case histogram: + b := k.hour.Histograms[field(x)] + if len(b) != len(v) { + b = make([]int64, len(v)) + } + for i := range v { + b[i] += v[i] + if i < len(before) { + b[i] -= before[i] + } + } + k.hour.Histograms[field(x)] = b + } + } + k.last, k.at = cur, now +} + +func (k *Keeper) gauges(cur state) { + for x, v := range cur { + if x.kind == gauge { + k.hour.Gauges[field(x)] = k.hour.Gauges[field(x)].add(v[0]) + } + } +} + +func (k *Keeper) put(h Hour) { + if err := k.store.PutHour(h); err != nil && k.failed != nil { + k.failed(err) + } +} diff --git a/internal/metrics/hours_test.go b/internal/metrics/hours_test.go new file mode 100644 index 0000000..a357881 --- /dev/null +++ b/internal/metrics/hours_test.go @@ -0,0 +1,129 @@ +package metrics + +import ( + "errors" + "testing" + "time" +) + +type hours struct{ got []Hour } + +func (h *hours) PutHour(x Hour) error { + h.got = append(h.got, x) + return nil +} + +func at(hh, mm int) time.Time { return time.Date(2026, 10, 24, hh, mm, 0, 0, time.UTC) } + +// An hour holds each counter's rise, each gauge's low, high and mean over the +// minutes sampled, and each histogram's buckets; it goes to the store once +// the clock has left it, and Close hands over the one it is in. +func TestTheKeeperRollsTheSeriesUpIntoHours(t *testing.T) { + s := &Set{} + c := s.Counter("ftwcloud_drops_total", "", "reason", "x") + g := s.Gauge("ftwcloud_sockets", "", "kind", "box") + h := s.Histogram("ftwcloud_forward_seconds", "", "path", "tun_up") + c.Add(7) // before this process's keeper: not in its hours + g.Add(2) + + now := at(5, 10) + store := &hours{} + k := &Keeper{set: s, store: store, now: func() time.Time { return now }} + k.begin() + + c.Add(3) + now = at(5, 11) + k.sample() + g.Add(3) + for range 10 { + h.Observe(2 * time.Millisecond) + } + now = at(5, 12) + k.sample() + now = at(6, 0) + k.sample() + if len(store.got) != 1 { + t.Fatalf("%d hours handed over", len(store.got)) + } + five := store.got[0] + if five.Start != at(5, 0).UnixMilli() || five.Ms != (50*time.Minute).Milliseconds() || five.Starts != 1 { + t.Fatalf("hour %+v", five) + } + if five.Counters["drops_x"] != 3 { + t.Fatalf("counters %v", five.Counters) + } + if g := five.Gauges["sockets_box"]; g != (GaugeHour{Min: 2, Max: 5, Sum: 9, N: 3}) { + t.Fatalf("gauge %+v", g) + } + if b := five.Histograms["forward_seconds_tun_up"]; Quantile(b, 0.99) != 0.0025 || sum(b) != 10 { + t.Fatalf("buckets %v", b) + } + + c.Inc() + now = at(6, 30) + k.Close() + k.sample() + k.Close() + if len(store.got) != 2 { + t.Fatalf("%d hours handed over", len(store.got)) + } + six := store.got[1] + if six.Start != at(6, 0).UnixMilli() || six.Ms != (30*time.Minute).Milliseconds() || six.Starts != 0 || + six.Counters["drops_x"] != 1 || six.Gauges["sockets_box"] != (GaugeHour{Min: 5, Max: 5, Sum: 5, N: 1}) { + t.Fatalf("hour %+v", six) + } +} + +// A restart inside an hour adds the new process's part to the old one's. +func TestARestartAddsToTheHourItSplit(t *testing.T) { + a := newHour(at(5, 0)) + a.Ms, a.Starts = 40*60_000, 0 + a.Counters["tun_frames_up"] = 100 + a.Gauges["sockets_box"] = GaugeHour{Min: 90, Max: 100, Sum: 950, N: 10} + a.Histograms["handshake_seconds_box"] = []int64{1, 2, 0} + b := newHour(at(5, 0)) + b.Ms, b.Starts = 19*60_000, 1 + b.Counters["tun_frames_up"] = 40 + b.Counters["drains"] = 1 + b.Gauges["sockets_box"] = GaugeHour{Min: 0, Max: 99, Sum: 500, N: 10} + b.Gauges["homes"] = GaugeHour{Min: 3, Max: 3, Sum: 3, N: 1} + b.Histograms["handshake_seconds_box"] = []int64{0, 5, 1} + a.Merge(b) + if a.Ms != 59*60_000 || a.Starts != 1 || a.Counters["tun_frames_up"] != 140 || a.Counters["drains"] != 1 { + t.Fatalf("%+v", a) + } + if a.Gauges["sockets_box"] != (GaugeHour{Min: 0, Max: 100, Sum: 1450, N: 20}) || a.Gauges["homes"].N != 1 { + t.Fatalf("gauges %+v", a.Gauges) + } + if b := a.Histograms["handshake_seconds_box"]; b[0] != 1 || b[1] != 7 || b[2] != 1 { + t.Fatalf("buckets %v", b) + } +} + +// An hour the store refuses is reported, and the keeper goes on. +func TestAnHourTheStoreRefusesIsReported(t *testing.T) { + s := &Set{} + var heard error + now := at(5, 59) + k := &Keeper{set: s, store: refusing{}, failed: func(err error) { heard = err }, now: func() time.Time { return now }} + k.begin() + now = at(6, 0) + k.sample() + if heard == nil { + t.Fatal("the refusal went unheard") + } + if k.hour.Start != at(6, 0).UnixMilli() { + t.Fatal("the keeper stopped at the refusal") + } +} + +type refusing struct{} + +func (refusing) PutHour(Hour) error { return errors.New("disk full") } + +func sum(b []int64) (n int64) { + for _, c := range b { + n += c + } + return n +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index db3bc05..0ceee55 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -72,6 +72,9 @@ func (h *Histogram) Observe(d time.Duration) { // Since counts the time since t. func (h *Histogram) Since(t time.Time) { h.Observe(time.Since(t)) } +// Count is how many durations it has counted. +func (h *Histogram) Count() int64 { return h.n.Load() } + // snapshot is the bucket counts now. func (h *Histogram) snapshot() []int64 { out := make([]int64, len(h.counts)) @@ -83,7 +86,9 @@ func (h *Histogram) snapshot() []int64 { // quantile is the upper bound of the bucket that holds the q-th quantile of // counts, a delta between two snapshots, or 0 without any. -func (h *Histogram) quantile(counts []int64, q float64) float64 { +func (h *Histogram) quantile(counts []int64, q float64) float64 { return quantile(h.bounds, counts, q) } + +func quantile(bounds []float64, counts []int64, q float64) float64 { var total int64 for _, c := range counts { total += c @@ -96,10 +101,10 @@ func (h *Histogram) quantile(counts []int64, q float64) float64 { for i, c := range counts { seen += c if seen >= rank { - if i == len(h.bounds) { + if i == len(bounds) { return math.Inf(1) } - return h.bounds[i] + return bounds[i] } } return math.Inf(1) @@ -270,6 +275,18 @@ func Sockets(kind string) *Gauge { return Default.Gauge("ftwcloud_sockets", "Open sockets, by kind.", "kind", kind) } +// TransportSockets is the open sockets of one kind over one transport, +// quic or websocket. +func TransportSockets(kind, transport string) *Gauge { + return Default.Gauge("ftwcloud_transport_sockets", "Open sockets, by kind and transport.", "kind", kind, "transport", transport) +} + +// BytesOut counts the bytes of the messages the service writes to one kind +// of socket. TLS, WebSocket and QUIC framing come on top. +func BytesOut(kind string) *Counter { + return Default.Counter("ftwcloud_bytes_out_total", "Bytes of messages written to sockets, by kind; TLS, WebSocket and QUIC framing not counted.", "kind", kind) +} + // Handshake times one kind of handshake, from the upgrade or the request to // the answer that ends it. func Handshake(kind string) *Histogram { diff --git a/internal/ops/ops.go b/internal/ops/ops.go new file mode 100644 index 0000000..59cfa88 --- /dev/null +++ b/internal/ops/ops.go @@ -0,0 +1,278 @@ +// Package ops is Sourceful's internal view of the service, for running it and +// for understanding the fleet. It keeps the service's own hours (package +// metrics) for 90 days, and serves them with the exact daily totals of the +// anonymous fleet and energy reports on GET /v1/ops to one key: the one the +// project stats Worker holds. +// +// Nothing in it names a home, a device or an address. Every hour sums over +// the whole process, as every series does. The days are the totals that +// /v1/fleet/public and /v1/energy/public hold to a group size, which keep no +// report; here they come exact, but only once closed, so that two answers +// never differ by one report. +package ops + +import ( + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "math" + "net/http" + "strings" + "time" + + "github.com/srcfl/ftw-cloud/internal/metrics" + "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" +) + +// Path is where the feed answers. +const Path = "/v1/ops" + +const ( + Schema = "ftw.cloud-ops/1" + Meaning = "sums over the whole service, and closed daily totals over all reports; no home, device or address" + // MaxHours is the most hours one answer carries: a week. + MaxHours = 168 + // settle is how long after its end an hour may still grow: the next + // process adds the rest of an hour a restart split at its first sample. + settle = 5 * time.Minute + // defaultBack is how far back an answer starts without since. + defaultBack = 48 * time.Hour +) + +// Service keeps the hours and serves the feed. +type Service struct { + reg *registry.Registry + key []byte // SHA-256 of the key; nil serves no one + shard string + now func() time.Time +} + +// New keeps hours in reg. keySHA256 is the hex SHA-256 of the key the feed +// answers to, which is no secret: it does not help anyone make the key. +// Empty serves no one, and the hours are kept all the same. +func New(reg *registry.Registry, keySHA256, shard string) (*Service, error) { + if err := reg.OpsInit(); err != nil { + return nil, err + } + if err := reg.FleetInit(); err != nil { + return nil, err + } + if err := reg.EnergyInit(); err != nil { + return nil, err + } + if err := reg.OpsPrune(time.Now()); err != nil { + return nil, err + } + s := &Service{reg: reg, shard: shard, now: time.Now} + if keySHA256 != "" { + key, err := hex.DecodeString(keySHA256) + if err != nil || len(key) != sha256.Size { + return nil, errors.New("ops: the key's hash is not 64 hex digits") + } + s.key = key + } + return s, nil +} + +// PutHour adds h to the hour kept for its start. +func (s *Service) PutHour(h metrics.Hour) error { + return s.reg.OpsPut(h.Start, s.now(), func(old []byte) ([]byte, error) { + if old == nil { + return json.Marshal(h) + } + var kept metrics.Hour + if err := json.Unmarshal(old, &kept); err != nil { + return nil, err + } + kept.Merge(h) + return json.Marshal(kept) + }) +} + +// Prune drops hours past registry.OpsKept. +func (s *Service) Prune(now time.Time) error { return s.reg.OpsPrune(now) } + +// allowed reports whether r carries the key. +func (s *Service) allowed(r *http.Request) bool { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") || len(token) < 32 { + return false + } + sum := sha256.Sum256([]byte(token)) + return subtle.ConstantTimeCompare(sum[:], s.key) == 1 +} + +// Feed is the whole answer. +type Feed struct { + Schema string `json:"schema"` + Meaning string `json:"meaning"` + Shard string `json:"shard"` + ObservedAt string `json:"observed_at"` + Hours []HourView `json:"hours"` + FleetDays []fleetwire.Day `json:"fleet_days"` + // EnergyDays are the closed dates' energy totals, newest first: the + // date, reports and each of energywire.Fields. + EnergyDays []map[string]any `json:"energy_days"` +} + +// HourView is one settled hour as the feed carries it. +type HourView struct { + Hour string `json:"hour"` + Seconds int64 `json:"seconds"` + Starts int64 `json:"starts"` + Counters map[string]int64 `json:"counters"` + Gauges map[string]GaugeView `json:"gauges"` + LatencyMs map[string]LatencyView `json:"latency_ms"` +} + +// GaugeView is a gauge's low, mean and high over the hour's samples. +type GaugeView struct { + Min int64 `json:"min"` + Mean float64 `json:"mean"` + Max int64 `json:"max"` +} + +// LatencyView is how many durations an hour counted and the upper bounds of +// the buckets that hold its quantiles, in milliseconds. A quantile is null +// when there is none or it lies past the last bucket. +type LatencyView struct { + N int64 `json:"n"` + P50 *float64 `json:"p50"` + P95 *float64 `json:"p95"` + P99 *float64 `json:"p99"` +} + +func view(h metrics.Hour) HourView { + v := HourView{Hour: time.UnixMilli(h.Start).UTC().Format(time.RFC3339), Seconds: int64(math.Round(float64(h.Ms) / 1000)), + Starts: h.Starts, Counters: h.Counters, Gauges: map[string]GaugeView{}, LatencyMs: map[string]LatencyView{}} + if v.Counters == nil { + v.Counters = map[string]int64{} + } + for k, g := range h.Gauges { + if g.N > 0 { + v.Gauges[k] = GaugeView{Min: g.Min, Mean: math.Round(float64(g.Sum)/float64(g.N)*1000) / 1000, Max: g.Max} + } + } + for k, b := range h.Histograms { + var n int64 + for _, c := range b { + n += c + } + l := LatencyView{N: n} + if n > 0 { + l.P50, l.P95, l.P99 = ms(b, 0.5), ms(b, 0.95), ms(b, 0.99) + } + v.LatencyMs[k] = l + } + return v +} + +func ms(counts []int64, q float64) *float64 { + s := metrics.Quantile(counts, q) + if math.IsInf(s, 0) { + return nil + } + v := math.Round(s*1e6) / 1e3 + return &v +} + +// ServeHTTP answers GET /v1/ops?since= to the key, with the settled +// hours from since's hour on, at most MaxHours, and the fleet's days. +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if s.key == nil { + http.NotFound(w, r) + return + } + if !s.allowed(r) { + w.Header().Set("WWW-Authenticate", `Bearer realm="ftw-cloud-ops"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + now := s.now() + from := now.Add(-defaultBack) + if q := r.URL.Query().Get("since"); q != "" { + t, err := time.Parse(time.RFC3339, q) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + from = t + } + from = maxTime(from, now.Add(-registry.OpsKept)).UTC().Truncate(time.Hour) + // An hour is served once it can no longer grow. + until := now.Add(-settle).UTC().Truncate(time.Hour).Add(-time.Hour) + rows, err := s.reg.OpsHours(from.UnixMilli(), until.Add(time.Hour).UnixMilli(), MaxHours) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + fleet, err := s.reg.FleetView(now) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + energy, err := s.energyDays(now) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + feed := Feed{Schema: Schema, Meaning: Meaning, Shard: s.shard, ObservedAt: now.UTC().Format(time.RFC3339), Hours: []HourView{}, FleetDays: []fleetwire.Day{}, EnergyDays: energy} + // A fleet day closes at its end: until then each report adds to it. + today := now.UTC().Format(time.DateOnly) + for _, d := range fleet.Days { + if d.Date < today { + feed.FleetDays = append(feed.FleetDays, d) + } + } + for _, raw := range rows { + var h metrics.Hour + if err := json.Unmarshal(raw, &h); err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + feed.Hours = append(feed.Hours, view(h)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(feed) +} + +// energyDays are the closed dates of the energy reports, as +// /v1/energy/public names them, without its group size. +func (s *Service) energyDays(now time.Time) ([]map[string]any, error) { + today := now.UTC().Truncate(24 * time.Hour) + first, last := today.AddDate(0, 0, -registry.EnergyDays), today.AddDate(0, 0, -energywire.AcceptDays-1) + totals, err := s.reg.EnergyTotals(first.Format(time.DateOnly), last.Format(time.DateOnly)) + if err != nil { + return nil, err + } + days := []map[string]any{} + for d := last; !d.Before(first); d = d.AddDate(0, 0, -1) { + day, ok := totals[d.Format(time.DateOnly)] + if !ok || day.Reports == 0 { + continue + } + row := map[string]any{"date": day.Date, "reports": day.Reports} + for i, name := range energywire.Fields { + row[name] = day.Sums[i] + } + days = append(days, row) + } + return days, nil +} + +func maxTime(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go new file mode 100644 index 0000000..5fe86a8 --- /dev/null +++ b/internal/ops/ops_test.go @@ -0,0 +1,335 @@ +package ops + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "maps" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw-cloud/internal/fleet" + "github.com/srcfl/ftw-cloud/internal/metrics" + "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw/go/protocol/energywire" + "github.com/srcfl/ftw/go/protocol/fleetwire" +) + +const key = "a-test-key-of-forty-three-characters-000000" + +func hash(k string) string { + s := sha256.Sum256([]byte(k)) + return hex.EncodeToString(s[:]) +} + +var now = time.Date(2026, 10, 24, 6, 23, 0, 0, time.UTC) + +func newService(t *testing.T, keyHash string) (*Service, *registry.Registry) { + t.Helper() + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = reg.Close() }) + s, err := New(reg, keyHash, "s1") + if err != nil { + t.Fatal(err) + } + s.now = func() time.Time { return now } + return s, reg +} + +func get(s *Service, path, auth string) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodGet, path, nil) + if auth != "" { + r.Header.Set("Authorization", auth) + } + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + return w +} + +func feed(t *testing.T, s *Service, path string) Feed { + t.Helper() + w := get(s, path, "Bearer "+key) + if w.Code != http.StatusOK { + t.Fatalf("%s: %d", path, w.Code) + } + var f Feed + d := json.NewDecoder(w.Body) + d.DisallowUnknownFields() + if err := d.Decode(&f); err != nil { + t.Fatal(err) + } + return f +} + +func hourAt(t time.Time) metrics.Hour { + return metrics.Hour{Start: t.UnixMilli(), Ms: 3_600_000, Counters: map[string]int64{}, Gauges: map[string]metrics.GaugeHour{}, Histograms: map[string][]int64{}} +} + +// The feed answers only the key; without one configured it answers no one. +func TestTheFeedAnswersOnlyTheKey(t *testing.T) { + s, _ := newService(t, hash(key)) + for _, auth := range []string{"", "Bearer", "Bearer " + key[:31], "Bearer " + key + "x", "Basic " + key, key, "Bearer " + key} { + w := get(s, Path, auth) + if w.Code != http.StatusUnauthorized || w.Body.Len() != 0 || w.Header().Get("WWW-Authenticate") == "" { + t.Fatalf("%q: %d %q", auth, w.Code, w.Body.String()) + } + } + if w := get(s, Path, "bearer "+key); w.Code != http.StatusOK { + t.Fatalf("the key: %d", w.Code) + } + r := httptest.NewRequest(http.MethodPost, Path, nil) + r.Header.Set("Authorization", "Bearer "+key) + w := httptest.NewRecorder() + s.ServeHTTP(w, r) + if w.Code != http.StatusMethodNotAllowed { + t.Fatalf("POST: %d", w.Code) + } + + off, _ := newService(t, "") + if w := get(off, Path, "Bearer "+key); w.Code != http.StatusNotFound { + t.Fatalf("no key configured: %d", w.Code) + } + reg, _ := registry.Open(filepath.Join(t.TempDir(), "registry.db")) + defer reg.Close() + for _, bad := range []string{"00", strings.Repeat("g", 64), hash(key) + "00"} { + if _, err := New(reg, bad, "s1"); err == nil { + t.Fatalf("a hash of %q was taken", bad) + } + } +} + +// Two parts of an hour, a restart between them, are served as one hour: +// counters added, gauges' low, mean and high over both, latency quantiles +// from both parts' buckets. +func TestARestartInsideAnHourIsServedAsOneHour(t *testing.T) { + s, _ := newService(t, hash(key)) + five := time.Date(2026, 10, 24, 5, 0, 0, 0, time.UTC) + a := hourAt(five) + a.Ms = 2_400_000 + a.Counters["tun_bytes_down"] = 1000 + a.Gauges["sockets_box"] = metrics.GaugeHour{Min: 9, Max: 10, Sum: 95, N: 10} + b := hourAt(five) + b.Ms, b.Starts = 1_199_600, 1 + b.Counters["tun_bytes_down"] = 500 + b.Counters["drains"] = 1 + b.Gauges["sockets_box"] = metrics.GaugeHour{Min: 0, Max: 10, Sum: 25, N: 5} + bounds := len(metrics.LatencyBounds) + 1 + a.Histograms["handshake_seconds_box"] = make([]int64, bounds) + b.Histograms["handshake_seconds_box"] = make([]int64, bounds) + a.Histograms["handshake_seconds_box"][3] = 90 // 1 ms + b.Histograms["handshake_seconds_box"][9] = 10 // 100 ms + b.Histograms["forward_seconds_tun_up"] = make([]int64, bounds) + b.Histograms["forward_seconds_tun_up"][bounds-1] = 1 // past 10 s + if err := s.PutHour(a); err != nil { + t.Fatal(err) + } + if err := s.PutHour(b); err != nil { + t.Fatal(err) + } + f := feed(t, s, Path) + if f.Schema != Schema || f.Shard != "s1" || f.ObservedAt != "2026-10-24T06:23:00Z" || len(f.Hours) != 1 { + t.Fatalf("%+v", f) + } + h := f.Hours[0] + if h.Hour != "2026-10-24T05:00:00Z" || h.Seconds != 3600 || h.Starts != 1 || h.Counters["tun_bytes_down"] != 1500 || h.Counters["drains"] != 1 { + t.Fatalf("%+v", h) + } + if g := h.Gauges["sockets_box"]; g.Min != 0 || g.Max != 10 || g.Mean != 8 { + t.Fatalf("gauge %+v", g) + } + l := h.LatencyMs["handshake_seconds_box"] + if l.N != 100 || *l.P50 != 1 || *l.P95 != 100 || *l.P99 != 100 { + t.Fatalf("latency %+v", l) + } + if l := h.LatencyMs["forward_seconds_tun_up"]; l.N != 1 || l.P50 != nil || l.P99 != nil { + t.Fatalf("past the last bucket: %+v", l) + } +} + +// An hour is served once it can no longer grow, from since's hour on, at +// most a week of hours an answer, and never one older than 90 days. +func TestTheFeedServesSettledHoursInWeeks(t *testing.T) { + s, reg := newService(t, hash(key)) + first := now.Add(-registry.OpsKept - 2*time.Hour).Truncate(time.Hour) + for h := first; !h.After(now); h = h.Add(time.Hour) { + if err := reg.OpsPut(h.UnixMilli(), first, func([]byte) ([]byte, error) { return json.Marshal(hourAt(h)) }); err != nil { + t.Fatal(err) + } + } + f := feed(t, s, Path) + if len(f.Hours) != 48 || f.Hours[0].Hour != "2026-10-22T06:00:00Z" || f.Hours[47].Hour != "2026-10-24T05:00:00Z" { + t.Fatalf("default: %d hours, %s to %s", len(f.Hours), f.Hours[0].Hour, f.Hours[len(f.Hours)-1].Hour) + } + // At 06:23 the hour that ended at 06:00 is settled; at 06:03 it is not. + now = time.Date(2026, 10, 24, 7, 23, 0, 0, time.UTC) + if f := feed(t, s, Path); f.Hours[len(f.Hours)-1].Hour != "2026-10-24T06:00:00Z" { + t.Fatalf("at 07:23 the last hour is %s", f.Hours[len(f.Hours)-1].Hour) + } + now = time.Date(2026, 10, 24, 7, 3, 0, 0, time.UTC) + if f := feed(t, s, Path); f.Hours[len(f.Hours)-1].Hour != "2026-10-24T05:00:00Z" { + t.Fatalf("at 07:03 the last hour is %s", f.Hours[len(f.Hours)-1].Hour) + } + now = time.Date(2026, 10, 24, 6, 23, 0, 0, time.UTC) + f = feed(t, s, Path+"?since=2020-01-01T00:00:00Z") + if len(f.Hours) != MaxHours || f.Hours[0].Hour != now.Add(-registry.OpsKept).Truncate(time.Hour).Format(time.RFC3339) { + t.Fatalf("since long ago: %d hours from %s", len(f.Hours), f.Hours[0].Hour) + } + if f := feed(t, s, Path+"?since=2026-10-24T03:30:00Z"); len(f.Hours) != 3 || f.Hours[0].Hour != "2026-10-24T03:00:00Z" { + t.Fatalf("since 03:30: %+v", f.Hours) + } + if w := get(s, Path+"?since=yesterday", "Bearer "+key); w.Code != http.StatusBadRequest { + t.Fatalf("a bad since: %d", w.Code) + } +} + +// Hours older than 90 days go, at upkeep and at each write. +func TestHoursGoAfterNinetyDays(t *testing.T) { + s, reg := newService(t, hash(key)) + old := now.Add(-registry.OpsKept - time.Hour).Truncate(time.Hour) + keep := now.Add(-registry.OpsKept + time.Hour).Truncate(time.Hour) + for _, h := range []time.Time{old, keep} { + if err := reg.OpsPut(h.UnixMilli(), old, func([]byte) ([]byte, error) { return json.Marshal(hourAt(h)) }); err != nil { + t.Fatal(err) + } + } + count := func() int { + rows, err := reg.OpsHours(0, now.UnixMilli(), 1000) + if err != nil { + t.Fatal(err) + } + return len(rows) + } + if count() != 2 { + t.Fatal("both hours should be there before upkeep") + } + if err := s.Prune(now); err != nil || count() != 1 { + t.Fatalf("after upkeep: %d %v", count(), err) + } + if err := reg.OpsPut(old.UnixMilli(), old, func([]byte) ([]byte, error) { return json.Marshal(hourAt(old)) }); err != nil { + t.Fatal(err) + } + if err := s.PutHour(hourAt(now.Truncate(time.Hour))); err != nil || count() != 2 { + t.Fatalf("a write should drop the old hour: %d %v", count(), err) + } +} + +// The fleet's and the energy reports' days come exactly as the cloud holds +// them, below the public group size of five too, once closed: a fleet day at +// its end, an energy date once no report may name it. +func TestTheDaysAreExactOnceClosed(t *testing.T) { + s, reg := newService(t, hash(key)) + s.now = time.Now + fl, err := fleet.New(reg) + if err != nil { + t.Fatal(err) + } + raw := `{"schema":"ftw.fleet/1","ftw_version":"v0.155.1-beta.1","channel":"beta","drivers":["sungrow"],"battery_kwh":"5-15","price_zone":"SE3","install_age":"1-6m"}` + report, err := fleetwire.Decode([]byte(raw)) + if err != nil { + t.Fatal(err) + } + yesterday := time.Now().UTC().AddDate(0, 0, -1) + for range 2 { + if ok, err := reg.FleetPut(report, yesterday); !ok || err != nil { + t.Fatal(ok, err) + } + } + w := httptest.NewRecorder() + fl.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/fleet", strings.NewReader(raw))) + if w.Code != http.StatusNoContent { + t.Fatalf("report: %d", w.Code) + } + closed := time.Now().UTC().AddDate(0, 0, -energywire.AcceptDays-1).Format(time.DateOnly) + open := time.Now().UTC().AddDate(0, 0, -1).Format(time.DateOnly) + for _, r := range []energywire.Report{ + {Schema: energywire.Schema, Date: closed, PVGeneration: 12, GridImport: 30, GridExport: 4, BatteryCharge: 9, BatteryDischarge: 8}, + {Schema: energywire.Schema, Date: closed, PVGeneration: 3, GridImport: 10, VehicleCharge: 7}, + {Schema: energywire.Schema, Date: open, PVGeneration: 1}, + } { + if ok, err := reg.EnergyAdd(r); !ok || err != nil { + t.Fatal(ok, err) + } + } + + f := feed(t, s, Path) + if len(f.FleetDays) != 1 || f.FleetDays[0].Date != yesterday.Format(time.DateOnly) || f.FleetDays[0].Reports != 2 || + f.FleetDays[0].Drivers["sungrow"] != 2 || f.FleetDays[0].Zones["SE3"] != 2 { + t.Fatalf("fleet days %+v", f.FleetDays) + } + if len(f.EnergyDays) != 1 { + t.Fatalf("energy days %+v", f.EnergyDays) + } + want := map[string]any{"date": closed, "reports": 2.0, "pv_generation_kwh": 15.0, "grid_import_kwh": 40.0, "grid_export_kwh": 4.0, + "battery_charge_kwh": 9.0, "battery_discharge_kwh": 8.0, "vehicle_charge_kwh": 7.0} + if !maps.Equal(f.EnergyDays[0], want) { + t.Fatalf("energy day %v", f.EnergyDays[0]) + } + var public struct { + Days []map[string]any `json:"days"` + } + w = httptest.NewRecorder() + fl.Public(w, httptest.NewRequest(http.MethodGet, "/v1/fleet/public", nil)) + _ = json.Unmarshal(w.Body.Bytes(), &public) + for _, d := range public.Days { + if d["reports"] != nil { + t.Fatalf("the public view should hide days below five: %s", w.Body.String()) + } + } +} + +// The feed's top level and each hour hold exactly these fields; a new one +// is a change to review, here and in the Worker's parser. +func TestTheFeedHoldsOnlyItsFields(t *testing.T) { + s, _ := newService(t, hash(key)) + h := hourAt(time.Date(2026, 10, 24, 5, 0, 0, 0, time.UTC)) + h.Counters["drops_x"] = 1 + if err := s.PutHour(h); err != nil { + t.Fatal(err) + } + var top map[string]json.RawMessage + _ = json.Unmarshal(get(s, Path, "Bearer "+key).Body.Bytes(), &top) + if got := keys(top); !slices.Equal(got, []string{"energy_days", "fleet_days", "hours", "meaning", "observed_at", "schema", "shard"}) { + t.Fatalf("top level %v", got) + } + var hours []map[string]json.RawMessage + _ = json.Unmarshal(top["hours"], &hours) + if got := keys(hours[0]); !slices.Equal(got, []string{"counters", "gauges", "hour", "latency_ms", "seconds", "starts"}) { + t.Fatalf("hour %v", got) + } + if err := s.reg.EnergyInit(); err != nil { + t.Fatal(err) + } + if _, err := s.reg.EnergyAdd(energywire.Report{Schema: energywire.Schema, Date: "2026-10-20", GridImport: 1}); err != nil { + t.Fatal(err) + } + var energy struct { + Days []map[string]json.RawMessage `json:"energy_days"` + } + _ = json.Unmarshal(get(s, Path, "Bearer "+key).Body.Bytes(), &energy) + if len(energy.Days) != 1 || !slices.Equal(keys(energy.Days[0]), slices.Sorted(slices.Values(append([]string{"date", "reports"}, energywire.Fields...)))) { + t.Fatalf("energy day %v", energy.Days) + } + var day map[string]json.RawMessage + b, _ := json.Marshal(fleetwire.NewDay("2026-10-24")) + _ = json.Unmarshal(b, &day) + if got := keys(day); !slices.Equal(got, []string{"battery_kwh", "channels", "date", "drivers", "ftw_versions", "install_age", "price_zones", "reports"}) { + t.Fatalf("fleet day %v", got) + } +} + +func keys(m map[string]json.RawMessage) []string { + var out []string + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} diff --git a/internal/registry/ops.go b/internal/registry/ops.go new file mode 100644 index 0000000..5eb425a --- /dev/null +++ b/internal/registry/ops.go @@ -0,0 +1,68 @@ +package registry + +import ( + "database/sql" + "errors" + "time" +) + +// OpsKept is how long the service keeps its own hours (package metrics). +const OpsKept = 90 * 24 * time.Hour + +// OpsInit creates the service's own hours: one row an hour, each a sum over +// the whole service. There is no home column, and none may be added. +func (r *Registry) OpsInit() error { + _, err := r.db.Exec(`CREATE TABLE IF NOT EXISTS ops_hours (hour_ms INTEGER PRIMARY KEY, summary BLOB NOT NULL)`) + return err +} + +// OpsPut stores the hour that starts at hourMS as merge makes it from the +// one stored, nil if none, in one transaction, and drops hours past OpsKept. +func (r *Registry) OpsPut(hourMS int64, now time.Time, merge func(old []byte) ([]byte, error)) error { + tx, err := r.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var old []byte + err = tx.QueryRow(`SELECT summary FROM ops_hours WHERE hour_ms = ?`, hourMS).Scan(&old) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + data, err := merge(old) + if err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO ops_hours(hour_ms, summary) VALUES (?, ?) ON CONFLICT(hour_ms) DO UPDATE SET summary=excluded.summary`, hourMS, data); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM ops_hours WHERE hour_ms < ?`, now.Add(-OpsKept).UnixMilli()); err != nil { + return err + } + return tx.Commit() +} + +// OpsHours returns at most limit hours that start in [fromMS, toMS), oldest +// first. +func (r *Registry) OpsHours(fromMS, toMS int64, limit int) ([][]byte, error) { + rows, err := r.rd.Query(`SELECT summary FROM ops_hours WHERE hour_ms >= ? AND hour_ms < ? ORDER BY hour_ms LIMIT ?`, fromMS, toMS, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out [][]byte + for rows.Next() { + var data []byte + if err := rows.Scan(&data); err != nil { + return nil, err + } + out = append(out, data) + } + return out, rows.Err() +} + +// OpsPrune drops hours past OpsKept, including when none is written. +func (r *Registry) OpsPrune(now time.Time) error { + _, err := r.db.Exec(`DELETE FROM ops_hours WHERE hour_ms < ?`, now.Add(-OpsKept).UnixMilli()) + return err +} diff --git a/internal/server/ops_e2e_test.go b/internal/server/ops_e2e_test.go new file mode 100644 index 0000000..95c67f3 --- /dev/null +++ b/internal/server/ops_e2e_test.go @@ -0,0 +1,95 @@ +package server + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "maps" + "math" + "slices" + "testing" + + "github.com/srcfl/ftw-cloud/internal/metrics" + "github.com/srcfl/ftw-cloud/internal/ops" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// An hour of the service's counts, over a box on QUIC and a phone on a +// WebSocket that send a command through the tunnel, holds the traffic and +// names no home, device, key or address. Its fields are the series the +// service had before the traffic: none is made for a home. +func TestAnHourOfCountsNamesNobody(t *testing.T) { + s := newStack(t) + svc, err := ops.New(s.reg, "", "s1") + if err != nil { + t.Fatal(err) + } + before := &held{} + metrics.Default.NewKeeper(before, nil).Close() + keeper := metrics.Default.NewKeeper(svc, func(err error) { t.Error(err) }) + + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + var phone *tunPhone + eventually(t, "the phone is admitted", func() bool { + phone = s.dialPhone(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}, "") + return phone.conn != nil && phone.tr != nil + }) + phone.open(box.static.Public(), 7, nil) + if ack, res := phone.cmd(7, "0192f2a0-7c1e-7000-8000-0000000000a0"); ack != protocol.MsgCmdAck || res != protocol.CmdApplied { + t.Fatalf("ack %q, result %q", ack, res) + } + keeper.Close() + + rows, err := s.reg.OpsHours(0, math.MaxInt64, 10) + if err != nil || len(rows) == 0 { + t.Fatalf("%d hours, %v", len(rows), err) + } + var h metrics.Hour + if err := json.Unmarshal(rows[len(rows)-1], &h); err != nil { + t.Fatal(err) + } + if h.Counters["tun_frames_up"] == 0 || h.Counters["tun_frames_down"] == 0 || h.Histograms["handshake_seconds_box"] == nil { + t.Fatalf("the hour missed the traffic: %v", h.Counters) + } + if g := h.Gauges["transport_sockets_box_quic"]; g.N == 0 { + t.Fatalf("no box over QUIC in the hour: %v", h.Gauges) + } + boxKey, _ := cloudwire.PublicKeyBytesOf(&box.key.PublicKey) + for _, row := range rows { + for _, needle := range []string{box.home, cloudwire.DeviceID(key.Public()), base64.RawURLEncoding.EncodeToString(boxKey), + base64.RawURLEncoding.EncodeToString(key.Public()), base64.RawURLEncoding.EncodeToString(box.static.Public()), "127.0.0.1", "::1"} { + if bytes.Contains(row, []byte(needle)) { + t.Fatalf("an hour holds %q: %s", needle, row) + } + } + } + for name, fields := range map[string][]string{ + "counters": slices.Sorted(maps.Keys(h.Counters)), + "gauges": slices.Sorted(maps.Keys(h.Gauges)), + "histograms": slices.Sorted(maps.Keys(h.Histograms)), + } { + var want []string + switch name { + case "counters": + want = slices.Sorted(maps.Keys(before.got.Counters)) + case "gauges": + want = slices.Sorted(maps.Keys(before.got.Gauges)) + default: + want = slices.Sorted(maps.Keys(before.got.Histograms)) + } + if !slices.Equal(fields, want) { + t.Fatalf("%s changed with the traffic: %v, before %v", name, fields, want) + } + } +} + +type held struct{ got metrics.Hour } + +func (h *held) PutHour(x metrics.Hour) error { + h.got = x + return nil +} diff --git a/internal/server/server.go b/internal/server/server.go index aa50ba8..33ddaa5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -12,6 +12,8 @@ // /healthz liveness, and nothing about any home // /oauth/*, /.well-known/oauth-* the authorization server for assistants (package oauth) // /mcp the MCP server, for assistants with a token (package mcp) +// /v1/ops the service's own hours and the fleet's exact days, to the +// project stats Worker's key only (package ops) // // The same handler serves HTTP/3 (package quicdoor), where the sockets do not // open: they are WebSockets over TCP. @@ -42,7 +44,7 @@ const SourceArchive = "/usr/share/ftw-cloud/source.tar.gz" // Routes are every path the service answers. var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/fleet/public", - "/v1/ohttp", "/.well-known/ohttp-gateway", "/v1/energy/public", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) + "/v1/ohttp", "/.well-known/ohttp-gateway", "/v1/energy/public", "/v1/stats", "/v1/ops", "/source", "/healthz", "/mcp"}, oauth.Routes...) // devRoutes adds a local inspection route in binaries built with // -tags ftwclouddev. It is nil in every other build. @@ -67,6 +69,8 @@ type Server struct { Statistics http.Handler MCP http.Handler Log *slog.Logger + // Ops is Sourceful's internal feed, to one key; nil serves none. + Ops http.Handler // AltSvc, when set, is the Alt-Svc header that points TCP clients at // HTTP/3 (package quicdoor). It goes on plain responses only: a client // that learns HTTP/3 from a WebSocket upgrade might try the socket over @@ -118,6 +122,11 @@ func (s *Server) Handler() http.Handler { } else { mux.HandleFunc("/v1/stats", notYet) } + if s.Ops != nil { + mux.Handle("/v1/ops", s.Ops) + } else { + mux.HandleFunc("/v1/ops", notYet) + } mux.HandleFunc("/source", source) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") @@ -220,6 +229,11 @@ func (s *Server) Upkeep(ctx context.Context, every time.Duration) { s.Log.Warn("statistics upkeep failed") } } + if f, ok := s.Ops.(interface{ Prune(time.Time) error }); ok { + if err := f.Prune(time.Now()); err != nil { + s.Log.Warn("ops upkeep failed") + } + } if err := s.Registry.TokenExpire(time.Now()); err != nil { s.Log.Warn("token upkeep failed", "err", err) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index b3a5a43..c1f0d93 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -16,6 +16,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/appsession" "github.com/srcfl/ftw-cloud/internal/fleet" "github.com/srcfl/ftw-cloud/internal/gateway" + "github.com/srcfl/ftw-cloud/internal/ops" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/statistics" "github.com/srcfl/ftw-cloud/internal/store" @@ -51,6 +52,10 @@ func newServer(t *testing.T) (*Server, string) { if s.Gateway, err = gateway.New(reg, testGateway(t)); err != nil { t.Fatal(err) } + // The internal feed with a key set: a request without it reads nothing. + if s.Ops, err = ops.New(reg, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "s1"); err != nil { + t.Fatal(err) + } up.Statistics = true s.Statistics.(*statistics.Service).Enabled = true // A home with readings in it, held the way a box's uplink holds them. diff --git a/internal/uplink/uplink.go b/internal/uplink/uplink.go index 0856385..eb2760a 100644 --- a/internal/uplink/uplink.go +++ b/internal/uplink/uplink.go @@ -82,6 +82,7 @@ const ( // What the uplink counts, summed over all homes. var ( boxSockets = metrics.Sockets("box") + boxOver = map[string]*metrics.Gauge{cloudwire.TransportQUIC: metrics.TransportSockets("box", cloudwire.TransportQUIC), cloudwire.TransportWebSocket: metrics.TransportSockets("box", cloudwire.TransportWebSocket)} boxHandshake = metrics.Handshake("box") tunDown = metrics.Forward("tun_down") wrongShard = metrics.WrongShard("box") @@ -97,6 +98,7 @@ var ( dropConnect = metrics.Drop("box_connect_rate") dropXBusy = metrics.Drop("x_in_flight") dropXUnknown = metrics.Drop("x_unknown") + boxBytesOut = metrics.BytesOut("box") boxesFollow = metrics.Default.Gauge("ftwcloud_boxes_following_shards", "Connected boxes that list shard in hello.caps.") ) @@ -308,6 +310,10 @@ func (u *Uplink) ServeConn(conn wire.Conn, ekm []byte) { conn.SetReadLimit(maxBoxFrame) boxSockets.Add(1) defer boxSockets.Add(-1) + if over := boxOver[wire.Transport(conn)]; over != nil { + over.Add(1) + defer over.Add(-1) + } sock := u.sockets.Add(conn) defer u.sockets.Remove(sock) c := &boxConn{u: u, conn: conn, sock: sock, ekm: ekm, started: time.Now()} @@ -607,8 +613,10 @@ func (c *boxConn) writeFrame(frame []byte) error { } if err != nil { _ = c.conn.Close() + return err } - return err + boxBytesOut.Add(int64(len(frame))) + return nil } // closeAndWait ends the uplink with a close code and reason, and waits for diff --git a/internal/webpush/webpush.go b/internal/webpush/webpush.go index dae0deb..ed3b2e0 100644 --- a/internal/webpush/webpush.go +++ b/internal/webpush/webpush.go @@ -17,11 +17,21 @@ import ( "strings" "time" + "github.com/srcfl/ftw-cloud/internal/metrics" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/cloudwire" ) +// Push services' answers per browser, summed over every home. +var results = map[string]*metrics.Counter{} + +func init() { + for _, r := range []string{"ok", "refused", "retry"} { + results[r] = metrics.Default.Counter("ftwcloud_webpush_total", "Push services' answers per browser, by result.", "result", r) + } +} + type Service struct { reg *registry.Registry client *http.Client @@ -172,17 +182,22 @@ func (s *Service) Send(ctx context.Context, home string) (took int, retry bool) req.Header.Set("Authorization", row.Auth) resp, e := s.client.Do(req) if e != nil { + results["retry"].Inc() retry = true continue } _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024)) _ = resp.Body.Close() if resp.StatusCode == 429 || resp.StatusCode >= 500 { + results["retry"].Inc() retry = true continue } if resp.StatusCode >= 200 && resp.StatusCode < 300 { + results["ok"].Inc() took++ + } else { + results["refused"].Inc() } // Terminal refusals (including expired subscriptions) do not repeat. if s.reg.WebPushMark(home, due, mark) != nil { diff --git a/internal/wire/wire.go b/internal/wire/wire.go index 6e04271..0b5950a 100644 --- a/internal/wire/wire.go +++ b/internal/wire/wire.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/srcfl/ftw/go/protocol/cloudwire" ) // Conn is a message connection: gorilla/websocket's method set. @@ -63,3 +64,12 @@ func Kind(conn Conn, want int) int { func Data(messageType int) bool { return messageType == websocket.BinaryMessage || messageType == Bulk || messageType == Datagram } + +// Transport is the registry's name for conn's transport: what a Carrier +// reports, and websocket for a plain WebSocket. +func Transport(conn Conn) string { + if c, ok := conn.(Carrier); ok { + return c.Transport() + } + return cloudwire.TransportWebSocket +} diff --git a/privacy/internal-operations.md b/privacy/internal-operations.md new file mode 100644 index 0000000..8250458 --- /dev/null +++ b/privacy/internal-operations.md @@ -0,0 +1,83 @@ +# Intern driftstatistik för FTW Cloud + +24 oktober 2026. Beskriver vad koden gör. Inte juridiskt granskad. + +Sourceful driver FTW Cloud och behöver förstå driften, felen och flottan för +att kunna förbättra tjänsten. Fredrik bad om det 24 oktober 2026: statistik +för internt bruk, samlad på ett ställe, utan ett nytt system. Den visas i den +privata vyn på stats.ftw.energy. + +## Vad som sparas och hur länge + +| Uppgift | Var | Hur länge | Vem når den | +|---|---|---|---| +| Tjänstens timsummor | Cloud-registret, `ops_hours` | 90 dagar | Workern, med sin nyckel | +| Samma timmar | Workerns D1, `cloud_ops_hours` | 90 dagar | Privata vyn, bakom Access | +| Exakta summor för stängda dagar med flottrapporter | Cloud: `fleet_days`, 90 dagar. D1: senaste vyn | Högst 90 dagar | Privata vyn | +| Exakta summor för stängda datum med energirapporter | Cloud: `energy_days`, 30 dagar. D1: senaste vyn | 30 dagar i Cloud, senaste vyn i D1 | Privata vyn | +| `/metrics` och minutraden, oförändrade | Minnet och Docker-loggen | Till nästa driftsättning | Den som har tillgång till värden | + +En timme håller hur mycket varje räknare steg, varje mätares lägsta, högsta +och medelvärde, varje fördröjnings fördelning, hur stor del av timmen en +process körde och hur många processer som startade. Serierna är de som +`/metrics` redan har: uttag per slag och transport, hem i minnet och +anslutna, byte till boxar och telefoner, tunnelns ramar och byte, tappade +meddelanden per skäl, Apples och webbläsarnas svar på notiser, MCP-anrop per +utfall och deras tid, handskakningar, dräneringar och Go-körtiden. + +En flottdag visas när den är slut, ett energidatum när ingen rapport längre +får nämna det. Därför skiljer sig två hämtningar aldrig med en enda rapport. + +## Vem ser den + +Bara den som Cloudflare Access släpper in på `stats.ftw.energy/admin*` och +`/api/admin*`. Policyn tar med GitHub-organisationen `srcfl`. Workern prövar +dessutom tokenets signatur, utfärdare och publik, så ett fel i Access-regeln +stänger i stället för att öppna. + +`GET /v1/ops` svarar bara den nyckel Workern har som hemlighet +(`CLOUD_OPS_KEY`). Cloud har bara nyckelns SHA-256, som inte är hemlig. Utan +den svarar vägen 404 till alla. Det finns ingen offentlig väg till +timmarna och ingen väg från dem till ett hem. + +## Varför det inte är personuppgifter + +- Varje serie summerar över hela processen. En etikett är ett fast ord i + koden, aldrig ett hem, en enhet, en nyckel eller en adress. + `TestCountsNameNobody` håller listan över tillåtna etiketter, och + `TestAnHourOfCountsNamesNobody` visar att trafik från en box och en + telefon inte skapar någon ny serie och inte lämnar deras id, nycklar eller + adress i timmen. +- Den finaste tiden är en timme. Ingen rad gäller ett hem. +- Flott- och energidagarna är summor av rapporter utan id. Cloud sparar + ingen enskild rapport. Flottrapporten räknas per dimension var för sig, + aldrig som kombination. Energirapporten går genom ett oberoende OHTTP-relä, + så Cloud ser aldrig avsändarens adress. +- Workern tar bara de fält flödet `ftw.cloud-ops/1` namnger och stoppar allt + annat innan något sparas (`stats/src/ops.ts` i srcfl/ftw-webapp). + +## Det som återstår + +Så länge flottan är liten kan en summa ändå säga något om ett fåtal kända +hushåll. En timme med en telefon ansluten klockan tre på natten, eller ett +stängt datum med en enda rapport, beskriver då någon som personalen kanske +känner. Fredrik valde 24 oktober 2026 att visa dagarna exakt internt, utan +den offentliga gruppstorleken fem. Det här är det valets pris. + +Det som håller risken nere: bara Access-skyddad personal ser vyn, inget fält +pekar ut ett hem, tiden är grov, dagar visas först när de är stängda, och +personal ska inte försöka koppla en summa till ett hushåll. Om risken ska +minska mer kan små dagar visas först när de är äldre, eller slås ihop till +veckor. Det är ett beslut för Fredrik. + +## Per-hem-data + +Den stannar där den är: i minnet för hemmets egna telefoner, och i +supportåtkomst bara när ägaren slår på den för ett ärende. Driftstatistiken +läser den inte och ger ingen väg dit. + +## Radering och rättigheter + +Det finns ingen rad per person att lämna ut eller radera. Timmar raderas +efter 90 dagar: i Cloud vid varje skrivning och vid underhåll, i D1 vid +Workerns timkörning. Dagssummorna följer sina egna gränser ovan. From fa879b063976edbcc0bba7c071fcf86e0a26586b Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 24 Oct 2026 09:03:26 +0200 Subject: [PATCH 8/8] fix(ops): a bad key hash leaves the feed off instead of the service down A malformed FTW_CLOUD_OPS_KEY_SHA256, such as a stray quote or line end in compose.env, made ops.New fail, and with it the whole service. The hash is now read by UseKey after New: quotes and white space around it are dropped, and a hash that is still not 64 hex digits is logged and leaves /v1/ops answering 404, while boxes and phones are served as before. Co-Authored-By: Claude Opus 5.5 --- cmd/ftwcloudd/main.go | 5 ++++- deploy/README.md | 3 ++- internal/ops/ops.go | 35 ++++++++++++++++++++++----------- internal/ops/ops_test.go | 23 +++++++++++++++++----- internal/server/ops_e2e_test.go | 2 +- internal/server/server_test.go | 7 ++++++- 6 files changed, 54 insertions(+), 21 deletions(-) diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index f9c7690..239e65e 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -182,10 +182,13 @@ func serve(args []string) error { return err } // The service's own hours, kept 90 days for Sourceful's internal view. - opsSvc, err := ops.New(reg, *opsKey, shards.Self()) + opsSvc, err := ops.New(reg, shards.Self()) if err != nil { return err } + if err := opsSvc.UseKey(*opsKey); err != nil { + log.Error("ops-key-sha256 is not 64 hex digits; /v1/ops stays off, and the hours are kept", "err", err) + } tokens := token.New(st, reg, log) tokens.Shards = shards srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Gateway: gatewaySvc, Statistics: usageSvc, Ops: opsSvc, Log: log} diff --git a/deploy/README.md b/deploy/README.md index f8039be..b6c6194 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -230,7 +230,8 @@ closed port, so the spread's floor can drop to a few seconds. `FTW_CLOUD_OPS_KEY_SHA256=`; the next deploy serves the feed. The hash is no secret, the key is: it lives only in the Worker. Until the service has a hash the Worker logs `awaiting key on Cloud`; while the two differ, a - 401. To rotate, do all three again. + 401. A hash that is not 64 hex digits is logged at start and leaves the + feed off; the service runs. To rotate, do all three again. - What to watch: `forward_seconds_*_p99` (time inside the service), `drops_*` (any rise), `push_queued_alert`, `homes_connected`, `go_memory_bytes`. diff --git a/internal/ops/ops.go b/internal/ops/ops.go index 59cfa88..8c51d13 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -51,10 +51,9 @@ type Service struct { now func() time.Time } -// New keeps hours in reg. keySHA256 is the hex SHA-256 of the key the feed -// answers to, which is no secret: it does not help anyone make the key. -// Empty serves no one, and the hours are kept all the same. -func New(reg *registry.Registry, keySHA256, shard string) (*Service, error) { +// New keeps hours in reg. Until UseKey takes a key, the feed serves no one, +// and the hours are kept all the same. +func New(reg *registry.Registry, shard string) (*Service, error) { if err := reg.OpsInit(); err != nil { return nil, err } @@ -67,15 +66,27 @@ func New(reg *registry.Registry, keySHA256, shard string) (*Service, error) { if err := reg.OpsPrune(time.Now()); err != nil { return nil, err } - s := &Service{reg: reg, shard: shard, now: time.Now} - if keySHA256 != "" { - key, err := hex.DecodeString(keySHA256) - if err != nil || len(key) != sha256.Size { - return nil, errors.New("ops: the key's hash is not 64 hex digits") - } - s.key = key + return &Service{reg: reg, shard: shard, now: time.Now}, nil +} + +// UseKey makes the feed answer the key whose SHA-256 is keySHA256, in hex, +// which is no secret: it does not help anyone make the key. Spaces, line +// ends and quotes around it, as an env file may leave them, are dropped. +// Empty turns the feed off. A hash that is not 64 hex digits turns it off +// too and is an error to log: a typo in an internal setting must not stop +// the service. +func (s *Service) UseKey(keySHA256 string) error { + s.key = nil + h := strings.Trim(keySHA256, " \t\r\n\"'") + if h == "" { + return nil + } + key, err := hex.DecodeString(h) + if err != nil || len(key) != sha256.Size { + return errors.New("ops: the key's hash is not 64 hex digits") } - return s, nil + s.key = key + return nil } // PutHour adds h to the hour kept for its start. diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go index 5fe86a8..eabfbc9 100644 --- a/internal/ops/ops_test.go +++ b/internal/ops/ops_test.go @@ -36,10 +36,13 @@ func newService(t *testing.T, keyHash string) (*Service, *registry.Registry) { t.Fatal(err) } t.Cleanup(func() { _ = reg.Close() }) - s, err := New(reg, keyHash, "s1") + s, err := New(reg, "s1") if err != nil { t.Fatal(err) } + if err := s.UseKey(keyHash); err != nil { + t.Fatal(err) + } s.now = func() time.Time { return now } return s, reg } @@ -97,12 +100,22 @@ func TestTheFeedAnswersOnlyTheKey(t *testing.T) { if w := get(off, Path, "Bearer "+key); w.Code != http.StatusNotFound { t.Fatalf("no key configured: %d", w.Code) } - reg, _ := registry.Open(filepath.Join(t.TempDir(), "registry.db")) - defer reg.Close() - for _, bad := range []string{"00", strings.Repeat("g", 64), hash(key) + "00"} { - if _, err := New(reg, bad, "s1"); err == nil { + // A hash that is not one is an error to log, and the feed stays off; the + // service still starts. Quotes and a line end an env file leaves are + // not part of the hash. + for _, bad := range []string{"00", strings.Repeat("g", 64), hash(key) + "00", "'" + hash(key)[1:] + "'"} { + if err := s.UseKey(bad); err == nil { t.Fatalf("a hash of %q was taken", bad) } + if w := get(s, Path, "Bearer "+key); w.Code != http.StatusNotFound { + t.Fatalf("after %q the feed answers %d", bad, w.Code) + } + } + if err := s.UseKey(`"` + hash(key) + "\"\r\n"); err != nil { + t.Fatal(err) + } + if w := get(s, Path, "Bearer "+key); w.Code != http.StatusOK { + t.Fatalf("a quoted hash with a line end: %d", w.Code) } } diff --git a/internal/server/ops_e2e_test.go b/internal/server/ops_e2e_test.go index 95c67f3..d0a8848 100644 --- a/internal/server/ops_e2e_test.go +++ b/internal/server/ops_e2e_test.go @@ -22,7 +22,7 @@ import ( // service had before the traffic: none is made for a home. func TestAnHourOfCountsNamesNobody(t *testing.T) { s := newStack(t) - svc, err := ops.New(s.reg, "", "s1") + svc, err := ops.New(s.reg, "s1") if err != nil { t.Fatal(err) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index c1f0d93..90c4343 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -53,9 +53,14 @@ func newServer(t *testing.T) (*Server, string) { t.Fatal(err) } // The internal feed with a key set: a request without it reads nothing. - if s.Ops, err = ops.New(reg, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "s1"); err != nil { + opsSvc, err := ops.New(reg, "s1") + if err != nil { + t.Fatal(err) + } + if err := opsSvc.UseKey("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); err != nil { t.Fatal(err) } + s.Ops = opsSvc up.Statistics = true s.Statistics.(*statistics.Service).Enabled = true // A home with readings in it, held the way a box's uplink holds them.