feat: hold 48 hours of history tiles and answer tiles.get (C6)

README.md

@@ -22,7 +22,7 @@ to it yet.
   messages signed by that box's key.
 - Each home's newest message of each class lives in memory, as the exact bytes
   the box sent. Readings are dropped an hour after the box goes quiet and are
-  never written to disk.
+  never written to disk. History tiles stay longer, also in memory only.
 - What the service holds follows the newest roster's grant, never `hello`:
   the readable copy can turn on or off while the uplink stays up.
 - The registry (SQLite) holds the home id, the box key, the signed roster,
@@ -132,6 +132,29 @@ to it yet.
   The widgets then read `/v1/now`. At most 4 an hour per home, since Apple
   budgets widget reloads. A device holds at most 4 tokens of each kind; a
   new one pushes out its oldest.
+- **History tiles (C6).** A box whose owner's grant has `ftw.history.read`
+  sends its last 48 hours of 5-minute history as `tiles`: 12-hour tiles, each
+  closed one the same bytes the box serves over Noise, and the tile still
+  filling whole or as an append when a bucket closes. The service checks
+  each against the protocol's three shapes and the four series, refuses an
+  envelope over 4 KiB, and holds it in memory as the box signed it: per home
+  at most five closed tiles, 16 pieces of the filling one and 64 KiB, all
+  homes together at most 256 MiB. A closed tile is replaced only by one the
+  box sent later. A tile the box did not sign closes the uplink, as any such
+  message does; one of no shape is dropped and counted. `hello_ok.have`
+  names the closed tiles held, so a box that comes back sends only what is
+  missing, and the service sends `resync {what: [tiles]}`, at most once per
+  5 minutes, when a roster gives history back or room frees after a drop. A
+  phone with `ftw.history.read` asks with `tiles.get {have}`, gets the tiles
+  it lacks and then `tiles.end {n}`, and gets each new tile as it comes. The
+  snapshot, `/v1/app/once` and `/v1/now` carry none. Tiles are dated history,
+  so they outlive the hour after which readings go: each stays until 48
+  hours after it ends, up to 60 hours of history, while the roster is in
+  force. With the box away a phone can therefore read them for 18 to 24
+  hours. A revoke, a grant without history, a roster that runs out, a purge
+  and the 90-day expiry delete them. Assistants read history from the box,
+  never from these tiles; while the box is away, `history_query` says so and
+  when FTW Cloud last heard from it.
 - **Widget token.** A device asks for its extension token over its app
   session with `token.get`. A widget reads `GET /v1/now` with it: the newest
   `live`, `prices` and `hb` as the box signed them, and nothing else. The
@@ -309,7 +332,6 @@ Not built yet:
 
 | Step | What | Where |
 |---|---|---|
-| C6 | Tiles | `internal/server` |
 | C7–C8 | Statistics | implemented with collection off; review pending |
 
 ## How it holds a home

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/fxamacker/cbor/v2 v2.9.4
 	github.com/gorilla/websocket v1.5.3
 	github.com/quic-go/quic-go v0.63.0
-	github.com/srcfl/ftw/go/protocol v0.0.0-20261018181148-499d5de65104
+	github.com/srcfl/ftw/go/protocol v0.0.0-20261023161258-bac597f77f0a
 	golang.org/x/crypto v0.57.0
 	golang.org/x/net v0.58.0
 	golang.org/x/sys v0.48.0

go.sum

@@ -52,8 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78
 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/srcfl/ftw/go/protocol v0.0.0-20261018181148-499d5de65104 h1:nwfkV8mMubBLB+cx7mxFYrVT6Asgl8ctXWfjXci8MZU=
-github.com/srcfl/ftw/go/protocol v0.0.0-20261018181148-499d5de65104/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
+github.com/srcfl/ftw/go/protocol v0.0.0-20261023161258-bac597f77f0a h1:1Bxa1t8Kum4ECjMCogXw87M4RyfMJFJT7vWzCXLyw08=
+github.com/srcfl/ftw/go/protocol v0.0.0-20261023161258-bac597f77f0a/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
 github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
 github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=

internal/appsession/appsession.go

@@ -757,6 +757,12 @@ func (c *phoneConn) apply(frame []byte) (int, error) {
 		if c.s.Push != nil {
 			return 0, pushed(c.s.Push.Unregister(c.home, c.dev.Device(), r))
 		}
+	case cloudwire.MsgTilesGet:
+		var g cloudwire.TilesGet
+		if err := cloudwire.Unmarshal(env.B, &g); err != nil {
+			return cloudwire.CloseProtocol, err
+		}
+		return 0, c.tiles(g.Have)
 	case cloudwire.MsgTokenGet:
 		if c.s.Tokens == nil {
 			return 0, nil
@@ -770,12 +776,33 @@ func (c *phoneConn) apply(frame []byte) (int, error) {
 	return 0, nil
 }
 
+// tiles answers tiles.get: the box's tiles envelopes the phone lacks, as
+// the box signed them, then tiles.end with how many went. A device that may
+// not read tiles gets tiles.end {n: 0}.
+func (c *phoneConn) tiles(have map[string]string) error {
+	envs, err := c.dev.Tiles(have)
+	if err != nil {
+		return err
+	}
+	for _, env := range envs {
+		if err := c.reply(env); err != nil {
+			return err
+		}
+	}
+	return c.say(cloudwire.MsgTilesEnd, cloudwire.TilesEnd{N: len(envs)})
+}
+
 // say hands an answer to the session's goroutine to send.
 func (c *phoneConn) say(t string, body any) error {
 	env, err := cloudwire.Unsigned(t, body)
 	if err != nil {
 		return err
 	}
+	return c.reply(env)
+}
+
+// reply hands one envelope to the session's goroutine to send.
+func (c *phoneConn) reply(env []byte) error {
 	select {
 	case c.replies <- env:
 		return nil

internal/appsession/tiles_test.go

@@ -0,0 +1,197 @@
+package appsession
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw-cloud/internal/store"
+	"github.com/srcfl/ftw/go/protocol"
+	"github.com/srcfl/ftw/go/protocol/cloudwire"
+)
+
+type tileVectorFile struct {
+	BoxKey struct {
+		D    string `json:"d"`
+		Home string `json:"home"`
+	} `json:"boxKey"`
+	Tiles []struct {
+		Name     string `json:"name"`
+		TileID   string `json:"tileId"`
+		Etag     string `json:"etag"`
+		Envelope string `json:"envelope"`
+	} `json:"tiles"`
+	Have map[string][]string `json:"have"`
+	Get  struct {
+		Body string `json:"body"`
+	} `json:"tilesGet"`
+	End struct {
+		Envelope string `json:"envelope"`
+	} `json:"tilesEnd"`
+}
+
+// tileHarness is the harness on the vectors' box, at 17:16 on the vectors'
+// day, with ftw.history.read in the grant and the vectors' tiles held.
+func tileHarness(t *testing.T) (*harness, tileVectorFile) {
+	t.Helper()
+	var v tileVectorFile
+	readVectors(t, "cloud-tile-vectors.json", &v)
+	h := newHarness(t)
+	h.shift.Store(int64(time.Until(time.UnixMilli(1_791_806_400_000 + (5*60+16)*60_000))))
+	key, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), unhex(t, v.BoxKey.D))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pk, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey)
+	if err := h.reg.Claim(v.BoxKey.Home, pk, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	box, err := h.store.For(storePrincipalBox(v.BoxKey.Home))
+	if err != nil {
+		t.Fatal(err)
+	}
+	box.Connected()
+	h.boxKey, h.home, h.box = key, v.BoxKey.Home, box
+	h.setRoster(h.devices(1, 1), protocol.ScopeLiveRead, protocol.ScopeHistoryRead)
+	return h, v
+}
+
+// putTile holds one of the vectors' envelopes, as the uplink does once it
+// has verified and checked it.
+func (h *harness) putTile(t *testing.T, raw []byte) {
+	t.Helper()
+	env, err := cloudwire.DecodeEnvelope(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cloudwire.Verify(&h.boxKey.PublicKey, h.home, env) != nil {
+		t.Fatal("a vector does not verify")
+	}
+	var b cloudwire.Tiles
+	if err := cloudwire.Unmarshal(env.B, &b); err != nil || b.Check() != nil {
+		t.Fatalf("a vector is not a tile: %v", err)
+	}
+	if kept, _ := h.box.PutTile(raw, b); !kept {
+		t.Fatal("a vector tile was not held")
+	}
+}
+
+// tilesAnswer reads the answer to one tiles.get: the envelopes, then
+// tiles.end, which it returns.
+func (p *phone) tilesAnswer(t *testing.T) ([][]byte, cloudwire.TilesEnd) {
+	t.Helper()
+	var got [][]byte
+	for {
+		env, code := p.next(2 * time.Second)
+		if code != 0 {
+			t.Fatalf("closed with %d", code)
+		}
+		e, err := cloudwire.DecodeEnvelope(env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if e.T == cloudwire.MsgTilesEnd {
+			var end cloudwire.TilesEnd
+			if len(e.Sig) != 0 || cloudwire.Unmarshal(e.B, &end) != nil {
+				t.Fatal("tiles.end is not the service's own")
+			}
+			return got, end
+		}
+		got = append(got, env)
+	}
+}
+
+func TestTilesGetAnswersWhatThePhoneLacksAsTheBoxSignedIt(t *testing.T) {
+	h, v := tileHarness(t)
+	for _, tv := range v.Tiles {
+		h.putTile(t, unhex(t, tv.Envelope))
+	}
+	p := h.mustDial(dialOpts{key: h.owner, hz: 1})
+	if len(p.snap.Envelopes) != 0 {
+		t.Fatalf("message 2 carries %d envelopes", len(p.snap.Envelopes))
+	}
+
+	p.send(cloudwire.MsgTilesGet, cloudwire.TilesGet{})
+	got, end := p.tilesAnswer(t)
+	if end.N != 4 || len(got) != 4 {
+		t.Fatalf("tiles.get {} answered %d, tiles.end says %d", len(got), end.N)
+	}
+	for i, tv := range v.Tiles {
+		if !bytes.Equal(got[i], unhex(t, tv.Envelope)) {
+			t.Fatalf("envelope %d (%s) is not the box's bytes", i, tv.Name)
+		}
+	}
+
+	// The vectors' have holds the closed tile: the pieces come alone, as
+	// the vectors say.
+	var g cloudwire.TilesGet
+	if err := cloudwire.Unmarshal(unhex(t, v.Get.Body), &g); err != nil {
+		t.Fatal(err)
+	}
+	p.send(cloudwire.MsgTilesGet, g)
+	got, end = p.tilesAnswer(t)
+	if end.N != len(v.Have["answer"]) || len(got) != end.N {
+		t.Fatalf("with have, %d envelopes and n %d", len(got), end.N)
+	}
+	for i, name := range v.Have["answer"] {
+		if !bytes.Equal(got[i], unhex(t, v.Tiles[i+1].Envelope)) || v.Tiles[i+1].Name != name {
+			t.Fatalf("answer %d is not %s", i, name)
+		}
+	}
+}
+
+func TestATileGoesToTheOwnersSessionAsItComesAndNotToAViewers(t *testing.T) {
+	h, v := tileHarness(t)
+	owner := h.mustDial(dialOpts{key: h.owner, hz: 1})
+	viewer := h.mustDial(dialOpts{key: h.viewer, hz: 1})
+	whole := unhex(t, v.Tiles[1].Envelope)
+	h.putTile(t, whole)
+	env, code := owner.next(2 * time.Second)
+	if code != 0 || !bytes.Equal(env, whole) {
+		t.Fatal("the owner did not get the tile as the box signed it")
+	}
+	// The viewer may not read history: tiles.get answers nothing, and the
+	// tile never came.
+	viewer.send(cloudwire.MsgTilesGet, cloudwire.TilesGet{})
+	got, end := viewer.tilesAnswer(t)
+	if len(got) != 0 || end.N != 0 {
+		t.Fatalf("a viewer got %d tiles", len(got))
+	}
+}
+
+// The box went away hours ago: no readings, but the tiles of the last 48
+// hours still come, as the box dated them.
+func TestTilesComeWhileTheBoxIsAway(t *testing.T) {
+	h, v := tileHarness(t)
+	h.live(true)
+	for _, tv := range v.Tiles {
+		h.putTile(t, unhex(t, tv.Envelope))
+	}
+	h.box.Disconnected(1)
+	h.shift.Add(int64(3 * time.Hour))
+	h.store.Sweep()
+	p := h.mustDial(dialOpts{key: h.owner})
+	if len(p.snap.Envelopes) != 0 || p.snap.Online {
+		t.Fatalf("after three hours away the snapshot holds %d readings", len(p.snap.Envelopes))
+	}
+	p.send(cloudwire.MsgTilesGet, cloudwire.TilesGet{})
+	if got, end := p.tilesAnswer(t); end.N != 4 || len(got) != 4 {
+		t.Fatalf("away, tiles.get answered %d", len(got))
+	}
+}
+
+// tiles.end is the bytes srcfl/ftw froze.
+func TestTilesEndIsTheVector(t *testing.T) {
+	var v tileVectorFile
+	readVectors(t, "cloud-tile-vectors.json", &v)
+	env, err := cloudwire.Unsigned(cloudwire.MsgTilesEnd, cloudwire.TilesEnd{N: 3})
+	if err != nil || !bytes.Equal(env, unhex(t, v.End.Envelope)) {
+		t.Fatal("tiles.end differs from the vector")
+	}
+}
+
+func storePrincipalBox(home string) store.Principal {
+	return store.Principal{Kind: store.Box, Home: home}
+}

internal/mcp/agent.go

@@ -582,6 +582,17 @@ func (c *agentConn) send(ctx context.Context, t string, id *uint32, body any) er
 	return nil
 }
 
+// away says whether a home's box is away, and when the service last heard
+// from it, for an agent the home still admits.
+func (p *Pool) away(sub oauth.Subject) (time.Time, bool) {
+	h, err := p.store.For(store.Principal{Kind: store.Agent, Home: sub.Home, DeviceKey: sub.Key})
+	if err != nil {
+		return time.Time{}, false
+	}
+	online, seen := h.Presence()
+	return seen, !online
+}
+
 // take waits for the session's turn. At most maxWaiting calls wait.
 func (c *agentConn) take(ctx context.Context) (func(), error) {
 	c.mu.Lock()

internal/mcp/history_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/srcfl/ftw-cloud/internal/store"
 	"github.com/srcfl/ftw/go/protocol"
 )
 
@@ -124,3 +125,43 @@ func TestStepLabel(t *testing.T) {
 		}
 	}
 }
+
+// A box that is away: history_query says so, and when FTW Cloud last heard
+// from it. It reads nothing from FTW Cloud's copy instead.
+func TestHistoryFromABoxThatIsAwaySaysWhenItWasLastSeen(t *testing.T) {
+	w := newWorld(t)
+	b := w.connectBox()
+	a := w.connect(b, "")
+	if _, isErr, text := a.call("history_query", map[string]any{"range": "1h"}); isErr {
+		t.Fatalf("history_query with the box up: %s", text)
+	}
+
+	_ = b.conn.Close()
+	box, err := w.st.For(store.Principal{Kind: store.Box, Home: b.home})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitFor(t, func() bool { online, _ := box.Presence(); return !online })
+	_, seen := box.Presence()
+	_, isErr, text := a.call("history_query", map[string]any{"range": "1h"})
+	if !isErr || !strings.Contains(text, "The box is away") ||
+		!strings.Contains(text, seen.UTC().Format(time.RFC3339)) || !strings.Contains(text, "ago") {
+		t.Fatalf("history_query with the box away: %v %s", isErr, text)
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if n := strings.Count(strings.Join(b.asked, "\n"), protocol.MsgHistQuery+" "); n != 1 {
+		t.Fatalf("the box was asked hist.query %d times", n)
+	}
+}
+
+func TestAgoWords(t *testing.T) {
+	for d, want := range map[time.Duration]string{
+		10 * time.Second: "under a minute", time.Minute: "1 minute", 59 * time.Minute: "59 minutes",
+		3 * time.Hour: "3 hours", 47 * time.Hour: "47 hours", 72 * time.Hour: "3 days",
+	} {
+		if got := agoWords(d); got != want {
+			t.Errorf("%s: %q, want %q", d, got, want)
+		}
+	}
+}