From 7dedcbf40d88786fcac7bcc56eac1c271241062b Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 23 Oct 2026 15:56:16 +0200 Subject: [PATCH 1/5] feat(cloudwire): the tiles class, its three shapes and tiles.get A tiles body is the app protocol's hist.chunk with the common header: closed tiles match the box's Noise answer byte for byte, and the trailing tile travels whole or as appends that Lay puts back together. cloud-tile-vectors.json freezes one of each shape for FTW Cloud and the phone. Co-Authored-By: Claude Opus 5.5 --- docs/protocol-cloud.md | 92 ++++- go/protocol/cloudwire/allowlist.go | 7 +- .../testdata/cloud-tile-vectors.json | 177 ++++++++ go/protocol/cloudwire/tiles.go | 231 +++++++++++ go/protocol/cloudwire/tiles_test.go | 389 ++++++++++++++++++ 5 files changed, 875 insertions(+), 21 deletions(-) create mode 100644 go/protocol/cloudwire/testdata/cloud-tile-vectors.json create mode 100644 go/protocol/cloudwire/tiles.go create mode 100644 go/protocol/cloudwire/tiles_test.go diff --git a/docs/protocol-cloud.md b/docs/protocol-cloud.md index d5330cd9d..e4c724401 100644 --- a/docs/protocol-cloud.md +++ b/docs/protocol-cloud.md @@ -8,7 +8,9 @@ in its `testdata`: `cloud-vectors.json` for the uplink, [tunnel](#tunnel) messages and a pairing proof, and `cloud-once-vectors.json` for one [one-shot read](#one-shot-read), and `cloud-shard-vectors.json` for which hosts a client may follow and how it -reads a restart's spread ([Shards and restarts](#shards-and-restarts)). +reads a restart's spread ([Shards and restarts](#shards-and-restarts)), and +`cloud-tile-vectors.json` for the three shapes of a [tile](#tiles-ftwhistoryread) +and how they lay back together. `srcfl/ftw-cloud`, the phone and the watch replay them. `cloudwire.View` is the phone's [freshness](#freshness) rules in Go, and the cases listed there are its tests. @@ -360,7 +362,7 @@ with two different values, so neither side's check passes through it. |---|---|---| | `boxNonce` | bstr, 32 B | Echo | | `ekm` | bstr, 32 B | The service's exporter value for this TLS session | - | `have` | map text→text | Tile id to etag, for the tiles the service holds | + | `have` | map text→text | Tile id to etag, for the closed tiles the service holds | | `rosterSeq` | uint | The roster number the service holds | | `statsPolicy` | text, optional | The statistics policy this service is approved to ingest. Missing means no statistics; the box sends none | @@ -376,9 +378,10 @@ with two different values, so neither side's check passes through it. before this check passes.** A proxy that reads `hello` learns the box's public key and grant id, and cannot use them. 4. **Box → service:** a `roster`, at every connect (see [Roster](#roster)); - then one full message of each class the grant allows; then the tiles - missing from `have`. While the link waits for a purge, the box sends only - `bye` instead (see [Revocation](#revocation-and-deletion)). + then one full message of each class the grant allows; then the trailing + tile whole and the closed tiles missing from `have` (see + [`tiles`](#tiles-ftwhistoryread)). While the link waits for a purge, the + box sends only `bye` instead (see [Revocation](#revocation-and-deletion)). 5. After that the box sends on change, by the cadence rules. ## Classes @@ -617,16 +620,56 @@ when its own rules (enabled, cooldown) allow. Delivery is best effort. ### `tiles`: `ftw.history.read` -The body is the app protocol's `hist.chunk` body: `tileId`, `etag`, `res`, -`startMs`, `stepMs`, `series`, `data` and `partial`. A closed tile has the -same tile id, etag and bytes as the box serves over Noise. +The last 48 hours of the box's 5-minute history, so a phone can draw it while +the box cannot answer. The body is the app protocol's `hist.chunk` body: -- `res` is `5m`. The service keeps 48 hours. -- `series` is drawn from `grid_w`, `pv_w`, `battery_w` and `load_w`, which the - box serves today, and `ev_w` and `battery_soc`, which C2 adds. -- When a 5-minute bucket closes, the box appends to the trailing tile. Only - then does `startMs` differ from the Noise meaning: it is the first new - bucket, and `partial` is true. +| Field | Type | Meaning | +|---|---|---| +| `tileId` | text | `5m/1/`: the 12-hour tile, at stride 1, as the app protocol names it | +| `etag` | text | FNV-1a of `data`, as the app protocol computes it | +| `res` | text | `5m` | +| `startMs` | int | Wall time of the first point. The tile's start, except in an append | +| `stepMs` | uint | 300,000 | +| `series` | [text] | `grid_w`, `pv_w`, `battery_w` and `load_w`, in that order: the series the app asks for over Noise | +| `data` | bstr | int32 LE, one block per series, the app protocol's missing value where the box has no record | +| `partial` | bool | The tile is still filling | + +A body is one of three shapes. `cloudwire.Tiles.Check` holds them, and +`cloud-tile-vectors.json` has one of each. + +- **Closed:** `partial` false, `startMs` the tile's start, all 144 points. + It has the same tile id, etag and bytes as the box serves over Noise for a + query of those series at 5 minutes. +- **Whole:** `partial` true, `startMs` the tile's start, all 144 points, with + the missing value from the first bucket that has not closed. It replaces + every piece of that tile the receiver holds. +- **Append:** `partial` true, `startMs` a later bucket, and only the buckets + that closed since the last piece of that tile: `data` holds as many points + per series as fit. The receiver lays it over what it holds. + +The box counts a bucket closed 30 s after it ends, when the history writer +has committed it. Then: + +- It sends an append for the trailing tile, or the tile whole when no piece + of it has gone on this connection or after 11 appends, so the service holds + at most 12 pieces of it and the box rereads it once an hour. +- When the tile's last bucket closes it sends the closed tile instead, and + nothing for the next tile until a bucket of it closes. +- Once an hour it packs the closed tiles of the last 48 hours again and + sends any whose etag changed, as when the ledger repaired a gap. +- At each connect, after the full state, it sends the trailing tile whole + and every closed tile of the last 48 hours that `hello_ok.have` lacks at + that etag, newest first. `resync {what: [tiles]}` sends the tiles it names, + or all of them when it names none, whatever `have` says. + +The box sends tiles only while NTP has synced its clock, since tiles are +dated by wall time, and only under a grant with `ftw.history.read`. They +count against the byte budget: about 100 KB a day. + +`cloudwire.Lay` puts what a receiver holds back together: a closed tile as +it is, and the trailing tile from its newest whole piece with each later +append laid over it, in `seq` order. The trailing tile stays partial, and +its etag is never offered as held. ### `hb`: every device in the roster @@ -850,7 +893,7 @@ a lane 0 frame closes the uplink. A fuzz test covers it. | `challenge` | `{v, nonce}` | First message. Unsigned | | `hello_ok` | See Handshake | Signed by the service key | | `watch` | `{hz: 1}` | Raises the period to 1 s for 60 s. While any phone of the home looks, the service sends it at once, every 50 s after that, and again when the box connects. Without it the box returns to 5 s. The box ignores any other `hz` | -| `resync` | `{what, tiles}` | `what` holds `full`, `tiles` or `roster`; `tiles` lists tile ids. The box honours at most one per 5 minutes and ignores the rest | +| `resync` | `{what, tiles}` | `what` holds `full`, `tiles` or `roster`; `tiles` lists tile ids, and none means every tile of the last 48 hours. The box honours at most one per 5 minutes and ignores the rest | | `purged` | `{atMs, what, grantId, statsId}` | Signed by the service key. `what` is `home`, `copy` or `stats`. Sent once every deletion it names is done | | `tun` | `{dev, sid, d}` | A phone's own session with the box. See [Tunnel](#tunnel). Only to a box that lists `tun` in `hello.caps` | @@ -937,7 +980,7 @@ as it decides for the same session over the relay. | Field | Type | Meaning | |---|---|---| -| `envelopes` | [Envelope] | Box envelopes in rising `seq`, each the box's own CBOR map, unchanged: for each class the device may read, the newest one the service holds; the newest `live` that carried `sources`; and the newest `hb`. Empty after an hour without the box, and while the readable copy is off | +| `envelopes` | [Envelope] | Box envelopes in rising `seq`, each the box's own CBOR map, unchanged: for each class the device may read but `tiles`, the newest one the service holds; the newest `live` that carried `sources`; and the newest `hb`. Empty after an hour without the box, and while the readable copy is off. A phone asks for tiles with `tiles.get` | | `lastSeenMs` | int | Service wall time when it last heard from the box | | `online` | bool | The box's uplink is up now | | `more` | uint | How many envelopes did not fit in message 2. They follow it as ordinary frames, in rising `seq` | @@ -959,7 +1002,7 @@ the table. |---|---|---| | `watch` | Any device | `{hz}`: 1 when the phone looks at the home, 0 when it looks away. It holds until the next `watch` or the end of the session | | `tun` | A device with `tun` | `{sid, d}`. The phone's session with its box. See [Tunnel](#tunnel). The service answers `tun` the same way | -| `tiles.get` | Devices with `ftw.history.read` | `{have}`, tile id to etag. The service answers with the `tiles` envelopes the phone lacks | +| `tiles.get` | Devices with `ftw.history.read` | `{have}`, tile id to etag of the closed tiles the phone holds. The service answers with the `tiles` envelopes the phone lacks: each closed tile it holds that `have` does not name at that etag, and every piece of the trailing tile, oldest tile first and each tile's pieces in `seq` order. Then `tiles.end {n}`, where `n` counts them. It answers this while the box is away too, and a device without the scope gets `tiles.end {n: 0}` | | `token.get` | Any device | Answered with `token {token, expiresInS}`. A new token replaces the device's old one | | `push.register` | Any device | `{apnsToken, environment, kind, activityId, app}`. One registration per APNs token; a Live Activity and its push-to-start each register their own. `app` is `iphone`, the default when absent, or `watch`, and picks the APNs topic | | `push.unregister` | Any device | `{apnsToken}` | @@ -1495,7 +1538,8 @@ writes do not gain access. CGNAT is not a private network for this choice. deletes the home's unpublished rows before accepting another report. If the box is offline, this happens on reconnect. - **Silence:** after an hour without the box's socket, the service serves only - `lastSeenMs` and no readings. When the roster runs out, it closes sessions + `lastSeenMs`, no readings, and the tiles, each until 48 hours after its + end. A tile is dated history and never reads as now. When the roster runs out, it closes sessions and refuses reads. After 30 days it drops tokens and APNs registrations. After 90 days without a valid roster it deletes the home as in a purge. - **A new home** that sends no roster within 24 hours is deleted. @@ -1576,7 +1620,11 @@ applied. Per home, the service keeps in memory the newest envelope of each class, the newest `live` that carried `sources`, 48 hours of tiles and a one-hour ring of -events. It writes no live readings to disk; the statistics files hold the +events. For tiles that is at most five closed tiles and 16 pieces of the +trailing tile, each as the box signed it; a tile that fails +`cloudwire.Tiles.Check` is dropped and counted. All homes together hold at +most a fixed number of tile bytes, past which new tiles are dropped and +counted. `have` in `hello_ok` names the closed tiles held. It writes no live readings to disk; the statistics files hold the only readings it stores. Its registry holds the home id, the box key, the roster, suspensions, the current and deleted `statsId`s, APNs registrations and token hashes. @@ -1669,7 +1717,11 @@ constants, so `srcfl/ftw-cloud` need not spell them by hand. | Extension token | 7 days | | Alert age for buttons | 10 minutes | | `box.unreachable` after | 600 s | -| Readings served after silence | 1 hour | +| Readings served after silence | 1 hour; tiles until 48 hours after their end | +| Tiles | `5m`, 12-hour tiles, the last 48 hours: at most 5 closed and 16 pieces of the trailing one per home | +| Bucket counted closed | 30 s after it ends | +| Trailing tile sent whole | At each connect, after 11 appends, and when its first bucket closes | +| Closed tiles packed again | Every hour, sent when the etag changed | | Sessions closed | When the roster runs out | | Tokens and APNs dropped after | 30 days | | Home deleted after | 90 days without a roster; 24 hours for a new home without one | diff --git a/go/protocol/cloudwire/allowlist.go b/go/protocol/cloudwire/allowlist.go index f4b21e66f..6c43b3621 100644 --- a/go/protocol/cloudwire/allowlist.go +++ b/go/protocol/cloudwire/allowlist.go @@ -39,6 +39,10 @@ var Fields = map[string][]string{ MsgHB: {"clock", "srcRev", "liveSeq", "lpSeq", "ages"}, + // The app protocol's hist.chunk: power means over 5-minute buckets of + // the four series the app draws (TileSeries), and nothing else. + MsgTiles: {"tileId", "etag", "res", "startMs", "stepMs", "series", "data", "partial"}, + MsgEvent: {"eventId", "ts", "kind", "severity", "args", "src", "lp", "cars"}, MsgRoster: {"rosterSeq", "ttlS", "devices", "grant", "pending", "noise", "webpush"}, @@ -66,7 +70,7 @@ var Fields = map[string][]string{ } // Bodies maps each class the box sends to its body type. Classes the box does -// not send yet (plan, health, tiles) have no entry; the service treats their +// not send yet (plan, health) have no entry; the service treats their // bytes as opaque either way. tun is not here: it has no header and is not a // class, and Unheaded holds it. var Bodies = map[string]any{ @@ -76,6 +80,7 @@ var Bodies = map[string]any{ MsgCar: Cars{}, MsgPrices: Prices{}, MsgHB: HB{}, + MsgTiles: Tiles{}, MsgEvent: Event{}, MsgRoster: Roster{}, MsgBye: Bye{}, diff --git a/go/protocol/cloudwire/testdata/cloud-tile-vectors.json b/go/protocol/cloudwire/testdata/cloud-tile-vectors.json new file mode 100644 index 000000000..67ce8a6d5 --- /dev/null +++ b/go/protocol/cloudwire/testdata/cloud-tile-vectors.json @@ -0,0 +1,177 @@ +{ + "boxKey": { + "d": "6c50811aa47e688ecbb01b2c1e91aca4a2ad7c879526d1c9679ef3fd25560074", + "pk": "04cd7a7c09b7c1fed854fcba642c1da25b48df5bf3208c67bcc2158374011e8c84bf6490130896243c527842441c170a64252874fdf719aa61444e329fe6bc02b8", + "home": "fjzyr67buzpyncqmjliwgezvde" + }, + "series": [ + "grid_w", + "pv_w", + "battery_w", + "load_w" + ], + "tiles": [ + { + "name": "closed", + "tileId": "5m/1/41476", + "etag": "986e18cd", + "startMs": 1791763200000, + "points": 144, + "partial": false, + "seq": 20, + "body": "ac6372657362356d637365711464626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900200300001103000002030000f3020000e4020000d5020000c6020000b7020000a8020000990200008a0200007b0200006c0200005d0200004e0200003f02000030020000210200001202000003020000f4010000e5010000d6010000c7010000b8010000a90100009a0100008b0100007c0100006d0100005e0100004f0100004001000031010000220100001301000004010000f5000000e6000000d7000000c8000000b9000000aa0000009b0000008c0000007d0000006e0000005f000000500000004100000032000000230000001400000005000000f6ffffffe7ffffffd8ffffffc9ffffffbaffffffabffffff9cffffff8dffffff7effffff6fffffff60ffffff51ffffff42ffffff33ffffff24ffffff15ffffff06fffffff7feffffe8feffffd9feffffcafeffffbbfeffffacfeffff9dfeffff8efeffff7ffeffff70feffff61feffff52feffff43feffff34feffff25feffff16feffff07fefffff8fdffffe9fdffffdafdffffcbfdffffbcfdffffadfdffff9efdffff8ffdffff80fdffff71fdffff62fdffff53fdffff44fdffff35fdffff26fdffff17fdffff08fdfffff9fcffffeafcffffdbfcffffccfcffffbdfcffffaefcffff9ffcffff90fcffff81fcffff72fcffff63fcffff54fcffff45fcffff36fcffff27fcffff18fcffff09fcfffffafbffffebfbffffdcfbffffcdfbffffbefbffffaffbffffa0fbffff91fbffff82fbffff73fbffff64fbffff55fbffff46fbffff37fbffff28fbffff19fbffff0afbfffffbfaffffecfaffffddfaffffcefaffffbffaffff00000000b5ffffff6affffff1fffffffd4feffff8afeffff40fefffff6fdffffacfdffff63fdffff1afdffffd1fcffff89fcffff42fcfffffbfbffffb5fbffff70fbffff2bfbffffe7faffffa4faffff62faffff20faffffe0f9ffffa0f9ffff62f9ffff25f9ffffe8f8ffffadf8ffff73f8ffff3bf8ffff03f8ffffcdf7ffff98f7ffff64f7ffff32f7ffff01f7ffffd2f6ffffa4f6ffff78f6ffff4df6ffff24f6fffffcf5ffffd6f5ffffb1f5ffff8ef5ffff6df5ffff4ef5ffff30f5ffff14f5fffffaf4ffffe1f4ffffcaf4ffffb5f4ffffa2f4ffff91f4ffff81f4ffff74f4ffff68f4ffff5ef4ffff56f4ffff50f4ffff4bf4ffff49f4ffff48f4ffff49f4ffff4cf4ffff51f4ffff58f4ffff61f4ffff6cf4ffff78f4ffff86f4ffff96f4ffffa8f4ffffbcf4ffffd2f4ffffe9f4ffff02f5ffff1df5ffff3af5ffff58f5ffff78f5ffff9af5ffffbdf5ffffe2f5ffff09f6ffff31f6ffff5bf6ffff87f6ffffb3f6ffffe2f6ffff12f7ffff43f7ffff76f7ffffaaf7ffffdff7ffff16f8ffff4ef8ffff87f8ffffc1f8fffffdf8ffff39f9ffff77f9ffffb6f9fffff5f9ffff36faffff78faffffbafafffffefaffff42fbffff87fbffffcdfbffff13fcffff5afcffffa2fcffffeafcffff32fdffff7bfdffffc5fdffff0ffeffff59feffffa3feffffeefeffff38ffffff83ffffffceffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900100008d0100008a0100008701000084010000810100007e0100007b0100007801000075010000720100006f0100006c010000690100006601000063010000600100005d0100005a0100005701000054010000510100004e0100004b0100004801000045010000420100003f0100003c010000390100003601000033010000300100002d0100002a0100002701000024010000210100001e0100001b0100001801000015010000120100000f0100000c01000009010000060100000301000000010000fd000000fa000000f7000000f4000000f1000000ee000000eb000000e8000000e5000000e2000000df000000dc000000d9000000d6000000d3000000d0000000cd000000ca000000c7000000c4000000c1000000be000000bb000000b8000000b5000000b2000000af000000ac000000a9000000a6000000a3000000a00000009d0000009a0000009700000094000000910000008e0000008b0000008800000085000000820000007f0000007c000000790000007600000073000000700000006d0000006a0000006700000064000000610000005e0000005b0000005800000055000000520000004f0000004c000000490000004600000043000000400000003d0000003a0000003700000034000000310000002e0000002b0000002800000025000000220000001f0000001c000000190000001600000013000000100000000d0000000a000000070000000400000001000000fefffffffbfffffff8fffffff5fffffff2ffffffefffffffecffffffe9ffffffe6ffffffe3ffffffc2010000c5010000c7010000ca010000cc010000cf010000d1010000d4010000d6010000d9010000db010000de010000e0010000e3010000e5010000e8010000ea010000ed010000ef010000f2010000f4010000f7010000f9010000fc010000fe010000010200000302000006020000080200000b0200000d020000100200001202000015020000170200001a0200000000008000000080000000800000008000000080000000802b0200002e020000300200003302000035020000380200003a0200003d0200003f020000420200004402000047020000490200004c0200004e020000510200005302000056020000580200005b0200005d020000600200006202000065020000670200006a0200006c0200006f020000710200007402000076020000790200007b0200007e020000800200008302000085020000880200008a0200008d0200008f020000920200009402000097020000990200009c0200009e020000a1020000a3020000a6020000a8020000ab020000ad020000b0020000b2020000b5020000b7020000ba020000bc020000bf020000c1020000c4020000c6020000c9020000cb020000ce020000d0020000d3020000d5020000d8020000da020000dd020000df020000e2020000e4020000e7020000e9020000ec020000ee020000f1020000f3020000f6020000f8020000fb020000fd020000000300000203000005030000070300000a0300000c0300000f030000110300001403000016030000190300001b0300001e030000200300002303000025030000280300006465746167683938366531386364667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437366677616c6c4d731b000001a12ffc5b30677061727469616cf46773746172744d731b000001a12d68b80068757074696d654d7319eb28", + "signedInputSha256": "3a55085d157a3c0db66c9463ece48a18f6d956eeaa6e403e8cabdfabf4dc7731", + "sig": "ef58627fcdebc178aa551ec1a64223d45d22488c58d699f0c26d9db6a989249248ac33233b4aa8d5196b4d5ff71046b54f700fc07c71084e08be200eb2af25d5", + "envelope": "a361625909b2ac6372657362356d637365711464626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900200300001103000002030000f3020000e4020000d5020000c6020000b7020000a8020000990200008a0200007b0200006c0200005d0200004e0200003f02000030020000210200001202000003020000f4010000e5010000d6010000c7010000b8010000a90100009a0100008b0100007c0100006d0100005e0100004f0100004001000031010000220100001301000004010000f5000000e6000000d7000000c8000000b9000000aa0000009b0000008c0000007d0000006e0000005f000000500000004100000032000000230000001400000005000000f6ffffffe7ffffffd8ffffffc9ffffffbaffffffabffffff9cffffff8dffffff7effffff6fffffff60ffffff51ffffff42ffffff33ffffff24ffffff15ffffff06fffffff7feffffe8feffffd9feffffcafeffffbbfeffffacfeffff9dfeffff8efeffff7ffeffff70feffff61feffff52feffff43feffff34feffff25feffff16feffff07fefffff8fdffffe9fdffffdafdffffcbfdffffbcfdffffadfdffff9efdffff8ffdffff80fdffff71fdffff62fdffff53fdffff44fdffff35fdffff26fdffff17fdffff08fdfffff9fcffffeafcffffdbfcffffccfcffffbdfcffffaefcffff9ffcffff90fcffff81fcffff72fcffff63fcffff54fcffff45fcffff36fcffff27fcffff18fcffff09fcfffffafbffffebfbffffdcfbffffcdfbffffbefbffffaffbffffa0fbffff91fbffff82fbffff73fbffff64fbffff55fbffff46fbffff37fbffff28fbffff19fbffff0afbfffffbfaffffecfaffffddfaffffcefaffffbffaffff00000000b5ffffff6affffff1fffffffd4feffff8afeffff40fefffff6fdffffacfdffff63fdffff1afdffffd1fcffff89fcffff42fcfffffbfbffffb5fbffff70fbffff2bfbffffe7faffffa4faffff62faffff20faffffe0f9ffffa0f9ffff62f9ffff25f9ffffe8f8ffffadf8ffff73f8ffff3bf8ffff03f8ffffcdf7ffff98f7ffff64f7ffff32f7ffff01f7ffffd2f6ffffa4f6ffff78f6ffff4df6ffff24f6fffffcf5ffffd6f5ffffb1f5ffff8ef5ffff6df5ffff4ef5ffff30f5ffff14f5fffffaf4ffffe1f4ffffcaf4ffffb5f4ffffa2f4ffff91f4ffff81f4ffff74f4ffff68f4ffff5ef4ffff56f4ffff50f4ffff4bf4ffff49f4ffff48f4ffff49f4ffff4cf4ffff51f4ffff58f4ffff61f4ffff6cf4ffff78f4ffff86f4ffff96f4ffffa8f4ffffbcf4ffffd2f4ffffe9f4ffff02f5ffff1df5ffff3af5ffff58f5ffff78f5ffff9af5ffffbdf5ffffe2f5ffff09f6ffff31f6ffff5bf6ffff87f6ffffb3f6ffffe2f6ffff12f7ffff43f7ffff76f7ffffaaf7ffffdff7ffff16f8ffff4ef8ffff87f8ffffc1f8fffffdf8ffff39f9ffff77f9ffffb6f9fffff5f9ffff36faffff78faffffbafafffffefaffff42fbffff87fbffffcdfbffff13fcffff5afcffffa2fcffffeafcffff32fdffff7bfdffffc5fdffff0ffeffff59feffffa3feffffeefeffff38ffffff83ffffffceffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900100008d0100008a0100008701000084010000810100007e0100007b0100007801000075010000720100006f0100006c010000690100006601000063010000600100005d0100005a0100005701000054010000510100004e0100004b0100004801000045010000420100003f0100003c010000390100003601000033010000300100002d0100002a0100002701000024010000210100001e0100001b0100001801000015010000120100000f0100000c01000009010000060100000301000000010000fd000000fa000000f7000000f4000000f1000000ee000000eb000000e8000000e5000000e2000000df000000dc000000d9000000d6000000d3000000d0000000cd000000ca000000c7000000c4000000c1000000be000000bb000000b8000000b5000000b2000000af000000ac000000a9000000a6000000a3000000a00000009d0000009a0000009700000094000000910000008e0000008b0000008800000085000000820000007f0000007c000000790000007600000073000000700000006d0000006a0000006700000064000000610000005e0000005b0000005800000055000000520000004f0000004c000000490000004600000043000000400000003d0000003a0000003700000034000000310000002e0000002b0000002800000025000000220000001f0000001c000000190000001600000013000000100000000d0000000a000000070000000400000001000000fefffffffbfffffff8fffffff5fffffff2ffffffefffffffecffffffe9ffffffe6ffffffe3ffffffc2010000c5010000c7010000ca010000cc010000cf010000d1010000d4010000d6010000d9010000db010000de010000e0010000e3010000e5010000e8010000ea010000ed010000ef010000f2010000f4010000f7010000f9010000fc010000fe010000010200000302000006020000080200000b0200000d020000100200001202000015020000170200001a0200000000008000000080000000800000008000000080000000802b0200002e020000300200003302000035020000380200003a0200003d0200003f020000420200004402000047020000490200004c0200004e020000510200005302000056020000580200005b0200005d020000600200006202000065020000670200006a0200006c0200006f020000710200007402000076020000790200007b0200007e020000800200008302000085020000880200008a0200008d0200008f020000920200009402000097020000990200009c0200009e020000a1020000a3020000a6020000a8020000ab020000ad020000b0020000b2020000b5020000b7020000ba020000bc020000bf020000c1020000c4020000c6020000c9020000cb020000ce020000d0020000d3020000d5020000d8020000da020000dd020000df020000e2020000e4020000e7020000e9020000ec020000ee020000f1020000f3020000f6020000f8020000fb020000fd020000000300000203000005030000070300000a0300000c0300000f030000110300001403000016030000190300001b0300001e030000200300002303000025030000280300006465746167683938366531386364667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437366677616c6c4d731b000001a12ffc5b30677061727469616cf46773746172744d731b000001a12d68b80068757074696d654d7319eb2861746574696c6573637369675840ef58627fcdebc178aa551ec1a64223d45d22488c58d699f0c26d9db6a989249248ac33233b4aa8d5196b4d5ff71046b54f700fc07c71084e08be200eb2af25d5", + "frame": "010100000a06a361625909b2ac6372657362356d637365711464626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900200300001103000002030000f3020000e4020000d5020000c6020000b7020000a8020000990200008a0200007b0200006c0200005d0200004e0200003f02000030020000210200001202000003020000f4010000e5010000d6010000c7010000b8010000a90100009a0100008b0100007c0100006d0100005e0100004f0100004001000031010000220100001301000004010000f5000000e6000000d7000000c8000000b9000000aa0000009b0000008c0000007d0000006e0000005f000000500000004100000032000000230000001400000005000000f6ffffffe7ffffffd8ffffffc9ffffffbaffffffabffffff9cffffff8dffffff7effffff6fffffff60ffffff51ffffff42ffffff33ffffff24ffffff15ffffff06fffffff7feffffe8feffffd9feffffcafeffffbbfeffffacfeffff9dfeffff8efeffff7ffeffff70feffff61feffff52feffff43feffff34feffff25feffff16feffff07fefffff8fdffffe9fdffffdafdffffcbfdffffbcfdffffadfdffff9efdffff8ffdffff80fdffff71fdffff62fdffff53fdffff44fdffff35fdffff26fdffff17fdffff08fdfffff9fcffffeafcffffdbfcffffccfcffffbdfcffffaefcffff9ffcffff90fcffff81fcffff72fcffff63fcffff54fcffff45fcffff36fcffff27fcffff18fcffff09fcfffffafbffffebfbffffdcfbffffcdfbffffbefbffffaffbffffa0fbffff91fbffff82fbffff73fbffff64fbffff55fbffff46fbffff37fbffff28fbffff19fbffff0afbfffffbfaffffecfaffffddfaffffcefaffffbffaffff00000000b5ffffff6affffff1fffffffd4feffff8afeffff40fefffff6fdffffacfdffff63fdffff1afdffffd1fcffff89fcffff42fcfffffbfbffffb5fbffff70fbffff2bfbffffe7faffffa4faffff62faffff20faffffe0f9ffffa0f9ffff62f9ffff25f9ffffe8f8ffffadf8ffff73f8ffff3bf8ffff03f8ffffcdf7ffff98f7ffff64f7ffff32f7ffff01f7ffffd2f6ffffa4f6ffff78f6ffff4df6ffff24f6fffffcf5ffffd6f5ffffb1f5ffff8ef5ffff6df5ffff4ef5ffff30f5ffff14f5fffffaf4ffffe1f4ffffcaf4ffffb5f4ffffa2f4ffff91f4ffff81f4ffff74f4ffff68f4ffff5ef4ffff56f4ffff50f4ffff4bf4ffff49f4ffff48f4ffff49f4ffff4cf4ffff51f4ffff58f4ffff61f4ffff6cf4ffff78f4ffff86f4ffff96f4ffffa8f4ffffbcf4ffffd2f4ffffe9f4ffff02f5ffff1df5ffff3af5ffff58f5ffff78f5ffff9af5ffffbdf5ffffe2f5ffff09f6ffff31f6ffff5bf6ffff87f6ffffb3f6ffffe2f6ffff12f7ffff43f7ffff76f7ffffaaf7ffffdff7ffff16f8ffff4ef8ffff87f8ffffc1f8fffffdf8ffff39f9ffff77f9ffffb6f9fffff5f9ffff36faffff78faffffbafafffffefaffff42fbffff87fbffffcdfbffff13fcffff5afcffffa2fcffffeafcffff32fdffff7bfdffffc5fdffff0ffeffff59feffffa3feffffeefeffff38ffffff83ffffffceffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900100008d0100008a0100008701000084010000810100007e0100007b0100007801000075010000720100006f0100006c010000690100006601000063010000600100005d0100005a0100005701000054010000510100004e0100004b0100004801000045010000420100003f0100003c010000390100003601000033010000300100002d0100002a0100002701000024010000210100001e0100001b0100001801000015010000120100000f0100000c01000009010000060100000301000000010000fd000000fa000000f7000000f4000000f1000000ee000000eb000000e8000000e5000000e2000000df000000dc000000d9000000d6000000d3000000d0000000cd000000ca000000c7000000c4000000c1000000be000000bb000000b8000000b5000000b2000000af000000ac000000a9000000a6000000a3000000a00000009d0000009a0000009700000094000000910000008e0000008b0000008800000085000000820000007f0000007c000000790000007600000073000000700000006d0000006a0000006700000064000000610000005e0000005b0000005800000055000000520000004f0000004c000000490000004600000043000000400000003d0000003a0000003700000034000000310000002e0000002b0000002800000025000000220000001f0000001c000000190000001600000013000000100000000d0000000a000000070000000400000001000000fefffffffbfffffff8fffffff5fffffff2ffffffefffffffecffffffe9ffffffe6ffffffe3ffffffc2010000c5010000c7010000ca010000cc010000cf010000d1010000d4010000d6010000d9010000db010000de010000e0010000e3010000e5010000e8010000ea010000ed010000ef010000f2010000f4010000f7010000f9010000fc010000fe010000010200000302000006020000080200000b0200000d020000100200001202000015020000170200001a0200000000008000000080000000800000008000000080000000802b0200002e020000300200003302000035020000380200003a0200003d0200003f020000420200004402000047020000490200004c0200004e020000510200005302000056020000580200005b0200005d020000600200006202000065020000670200006a0200006c0200006f020000710200007402000076020000790200007b0200007e020000800200008302000085020000880200008a0200008d0200008f020000920200009402000097020000990200009c0200009e020000a1020000a3020000a6020000a8020000ab020000ad020000b0020000b2020000b5020000b7020000ba020000bc020000bf020000c1020000c4020000c6020000c9020000cb020000ce020000d0020000d3020000d5020000d8020000da020000dd020000df020000e2020000e4020000e7020000e9020000ec020000ee020000f1020000f3020000f6020000f8020000fb020000fd020000000300000203000005030000070300000a0300000c0300000f030000110300001403000016030000190300001b0300001e030000200300002303000025030000280300006465746167683938366531386364667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437366677616c6c4d731b000001a12ffc5b30677061727469616cf46773746172744d731b000001a12d68b80068757074696d654d7319eb2861746574696c6573637369675840ef58627fcdebc178aa551ec1a64223d45d22488c58d699f0c26d9db6a989249248ac33233b4aa8d5196b4d5ff71046b54f700fc07c71084e08be200eb2af25d5" + }, + { + "name": "whole", + "tileId": "5m/1/41477", + "etag": "7285c700", + "startMs": 1791806400000, + "points": 144, + "partial": true, + "seq": 21, + "body": "ac6372657362356d637365711564626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be0300000000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000806465746167683732383563373030667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1310f03b0677061727469616cf56773746172744d731b000001a12ffbe60068757074696d654d7319eb32", + "signedInputSha256": "8b0f2e108c7c018c10927f7b960b3644316fd9a3a692f2eac230024589884d84", + "sig": "b48eeac0518263ea7e8115b94f495e9d86a4fde190243a54bd7450f55b7e1136c2fed27664d4a9c4d89560526f429574ac8242d7303fb8ce5500dd3e5c4ed861", + "envelope": "a361625909b2ac6372657362356d637365711564626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be0300000000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000806465746167683732383563373030667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1310f03b0677061727469616cf56773746172744d731b000001a12ffbe60068757074696d654d7319eb3261746574696c6573637369675840b48eeac0518263ea7e8115b94f495e9d86a4fde190243a54bd7450f55b7e1136c2fed27664d4a9c4d89560526f429574ac8242d7303fb8ce5500dd3e5c4ed861", + "frame": "010100000a06a361625909b2ac6372657362356d637365711564626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461590900b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be0300000000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000806465746167683732383563373030667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1310f03b0677061727469616cf56773746172744d731b000001a12ffbe60068757074696d654d7319eb3261746574696c6573637369675840b48eeac0518263ea7e8115b94f495e9d86a4fde190243a54bd7450f55b7e1136c2fed27664d4a9c4d89560526f429574ac8242d7303fb8ce5500dd3e5c4ed861" + }, + { + "name": "append 17:00", + "tileId": "5m/1/41477", + "etag": "229cca34", + "startMs": 1791824400000, + "points": 1, + "partial": true, + "seq": 22, + "body": "ac6372657362356d637365711664626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461502cf7ffff000000002cffffffc00300006465746167683232396363613334667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a131139790677061727469616cf56773746172744d731b000001a1310e8e8068757074696d654d7319eb3c", + "signedInputSha256": "31b4596a56ad6722bc4367cddf42a01c5d998cc12da5c5ed9ef5c2afc357aa97", + "sig": "547fd8fbd017964d5800bd918b35cf40ec97889659cc93ca571d0b6d0fe9ce5aca4f10c706b790026902c13e77a470b783c01d10e24949214b9902187101f422", + "envelope": "a3616258c0ac6372657362356d637365711664626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461502cf7ffff000000002cffffffc00300006465746167683232396363613334667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a131139790677061727469616cf56773746172744d731b000001a1310e8e8068757074696d654d7319eb3c61746574696c6573637369675840547fd8fbd017964d5800bd918b35cf40ec97889659cc93ca571d0b6d0fe9ce5aca4f10c706b790026902c13e77a470b783c01d10e24949214b9902187101f422", + "frame": "010100000113a3616258c0ac6372657362356d637365711664626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b06464617461502cf7ffff000000002cffffffc00300006465746167683232396363613334667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a131139790677061727469616cf56773746172744d731b000001a1310e8e8068757074696d654d7319eb3c61746574696c6573637369675840547fd8fbd017964d5800bd918b35cf40ec97889659cc93ca571d0b6d0fe9ce5aca4f10c706b790026902c13e77a470b783c01d10e24949214b9902187101f422" + }, + { + "name": "append 17:05", + "tileId": "5m/1/41477", + "etag": "0044f57f", + "startMs": 1791824700000, + "points": 2, + "partial": true, + "seq": 23, + "body": "ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201df7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683030343466353766667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb46", + "signedInputSha256": "2988782538a7ec1f2d63315375a01d829ffd9dadf242a84e4730f1eebea80f76", + "sig": "720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5", + "envelope": "a3616258d1ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201df7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683030343466353766667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb4661746574696c6573637369675840720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5", + "frame": "010100000124a3616258d1ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201df7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683030343466353766667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb4661746574696c6573637369675840720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5" + } + ], + "forged": { + "name": "forged append", + "tileId": "5m/1/41477", + "etag": "86ff94ae", + "startMs": 1791824700000, + "points": 2, + "partial": true, + "seq": 23, + "body": "ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201cf7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683836666639346165667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb46", + "signedInputSha256": "", + "sig": "720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5", + "envelope": "a3616258d1ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201cf7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683836666639346165667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb4661746574696c6573637369675840720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5", + "frame": "010100000124a3616258d1ac6372657362356d637365711764626f6f7450b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0646461746158201cf7ffff0ef7ffff000000000000000029ffffff26ffffffc3030000c50300006465746167683836666639346165667365726965738466677269645f776470765f7769626174746572795f77666c6f61645f7766737465704d731a000493e06674696c6549646a356d2f312f34313437376677616c6c4d731b000001a1311cbf50677061727469616cf56773746172744d731b000001a13113226068757074696d654d7319eb4661746574696c6573637369675840720790a10b1eb0806b4cf20594c97745ab1f4edc3d5835e7188da3bb7a88efe46e832e9b867de8d539ed15f2165123a0087a158f9d128298b0f36667b3165cf5" + }, + "laid": [ + { + "of": [ + "closed" + ], + "tileId": "5m/1/41476", + "etag": "986e18cd", + "startMs": 1791763200000, + "partial": false, + "data": "200300001103000002030000f3020000e4020000d5020000c6020000b7020000a8020000990200008a0200007b0200006c0200005d0200004e0200003f02000030020000210200001202000003020000f4010000e5010000d6010000c7010000b8010000a90100009a0100008b0100007c0100006d0100005e0100004f0100004001000031010000220100001301000004010000f5000000e6000000d7000000c8000000b9000000aa0000009b0000008c0000007d0000006e0000005f000000500000004100000032000000230000001400000005000000f6ffffffe7ffffffd8ffffffc9ffffffbaffffffabffffff9cffffff8dffffff7effffff6fffffff60ffffff51ffffff42ffffff33ffffff24ffffff15ffffff06fffffff7feffffe8feffffd9feffffcafeffffbbfeffffacfeffff9dfeffff8efeffff7ffeffff70feffff61feffff52feffff43feffff34feffff25feffff16feffff07fefffff8fdffffe9fdffffdafdffffcbfdffffbcfdffffadfdffff9efdffff8ffdffff80fdffff71fdffff62fdffff53fdffff44fdffff35fdffff26fdffff17fdffff08fdfffff9fcffffeafcffffdbfcffffccfcffffbdfcffffaefcffff9ffcffff90fcffff81fcffff72fcffff63fcffff54fcffff45fcffff36fcffff27fcffff18fcffff09fcfffffafbffffebfbffffdcfbffffcdfbffffbefbffffaffbffffa0fbffff91fbffff82fbffff73fbffff64fbffff55fbffff46fbffff37fbffff28fbffff19fbffff0afbfffffbfaffffecfaffffddfaffffcefaffffbffaffff00000000b5ffffff6affffff1fffffffd4feffff8afeffff40fefffff6fdffffacfdffff63fdffff1afdffffd1fcffff89fcffff42fcfffffbfbffffb5fbffff70fbffff2bfbffffe7faffffa4faffff62faffff20faffffe0f9ffffa0f9ffff62f9ffff25f9ffffe8f8ffffadf8ffff73f8ffff3bf8ffff03f8ffffcdf7ffff98f7ffff64f7ffff32f7ffff01f7ffffd2f6ffffa4f6ffff78f6ffff4df6ffff24f6fffffcf5ffffd6f5ffffb1f5ffff8ef5ffff6df5ffff4ef5ffff30f5ffff14f5fffffaf4ffffe1f4ffffcaf4ffffb5f4ffffa2f4ffff91f4ffff81f4ffff74f4ffff68f4ffff5ef4ffff56f4ffff50f4ffff4bf4ffff49f4ffff48f4ffff49f4ffff4cf4ffff51f4ffff58f4ffff61f4ffff6cf4ffff78f4ffff86f4ffff96f4ffffa8f4ffffbcf4ffffd2f4ffffe9f4ffff02f5ffff1df5ffff3af5ffff58f5ffff78f5ffff9af5ffffbdf5ffffe2f5ffff09f6ffff31f6ffff5bf6ffff87f6ffffb3f6ffffe2f6ffff12f7ffff43f7ffff76f7ffffaaf7ffffdff7ffff16f8ffff4ef8ffff87f8ffffc1f8fffffdf8ffff39f9ffff77f9ffffb6f9fffff5f9ffff36faffff78faffffbafafffffefaffff42fbffff87fbffffcdfbffff13fcffff5afcffffa2fcffffeafcffff32fdffff7bfdffffc5fdffff0ffeffff59feffffa3feffffeefeffff38ffffff83ffffffceffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900100008d0100008a0100008701000084010000810100007e0100007b0100007801000075010000720100006f0100006c010000690100006601000063010000600100005d0100005a0100005701000054010000510100004e0100004b0100004801000045010000420100003f0100003c010000390100003601000033010000300100002d0100002a0100002701000024010000210100001e0100001b0100001801000015010000120100000f0100000c01000009010000060100000301000000010000fd000000fa000000f7000000f4000000f1000000ee000000eb000000e8000000e5000000e2000000df000000dc000000d9000000d6000000d3000000d0000000cd000000ca000000c7000000c4000000c1000000be000000bb000000b8000000b5000000b2000000af000000ac000000a9000000a6000000a3000000a00000009d0000009a0000009700000094000000910000008e0000008b0000008800000085000000820000007f0000007c000000790000007600000073000000700000006d0000006a0000006700000064000000610000005e0000005b0000005800000055000000520000004f0000004c000000490000004600000043000000400000003d0000003a0000003700000034000000310000002e0000002b0000002800000025000000220000001f0000001c000000190000001600000013000000100000000d0000000a000000070000000400000001000000fefffffffbfffffff8fffffff5fffffff2ffffffefffffffecffffffe9ffffffe6ffffffe3ffffffc2010000c5010000c7010000ca010000cc010000cf010000d1010000d4010000d6010000d9010000db010000de010000e0010000e3010000e5010000e8010000ea010000ed010000ef010000f2010000f4010000f7010000f9010000fc010000fe010000010200000302000006020000080200000b0200000d020000100200001202000015020000170200001a0200000000008000000080000000800000008000000080000000802b0200002e020000300200003302000035020000380200003a0200003d0200003f020000420200004402000047020000490200004c0200004e020000510200005302000056020000580200005b0200005d020000600200006202000065020000670200006a0200006c0200006f020000710200007402000076020000790200007b0200007e020000800200008302000085020000880200008a0200008d0200008f020000920200009402000097020000990200009c0200009e020000a1020000a3020000a6020000a8020000ab020000ad020000b0020000b2020000b5020000b7020000ba020000bc020000bf020000c1020000c4020000c6020000c9020000cb020000ce020000d0020000d3020000d5020000d8020000da020000dd020000df020000e2020000e4020000e7020000e9020000ec020000ee020000f1020000f3020000f6020000f8020000fb020000fd020000000300000203000005030000070300000a0300000c0300000f030000110300001403000016030000190300001b0300001e03000020030000230300002503000028030000" + }, + { + "of": [ + "whole" + ], + "tileId": "5m/1/41477", + "etag": "7285c700", + "startMs": 1791806400000, + "partial": true, + "data": "b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be030000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080" + }, + { + "of": [ + "whole", + "append 17:00", + "append 17:05" + ], + "tileId": "5m/1/41477", + "etag": "96002093", + "startMs": 1791806400000, + "partial": true, + "data": "b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff2cf7ffff1df7ffff0ef7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff2cffffff29ffffff26ffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be030000c0030000c3030000c5030000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080" + }, + { + "of": [ + "closed", + "whole", + "append 17:00", + "append 17:05" + ], + "tileId": "5m/1/41476", + "etag": "986e18cd", + "startMs": 1791763200000, + "partial": false, + "data": "200300001103000002030000f3020000e4020000d5020000c6020000b7020000a8020000990200008a0200007b0200006c0200005d0200004e0200003f02000030020000210200001202000003020000f4010000e5010000d6010000c7010000b8010000a90100009a0100008b0100007c0100006d0100005e0100004f0100004001000031010000220100001301000004010000f5000000e6000000d7000000c8000000b9000000aa0000009b0000008c0000007d0000006e0000005f000000500000004100000032000000230000001400000005000000f6ffffffe7ffffffd8ffffffc9ffffffbaffffffabffffff9cffffff8dffffff7effffff6fffffff60ffffff51ffffff42ffffff33ffffff24ffffff15ffffff06fffffff7feffffe8feffffd9feffffcafeffffbbfeffffacfeffff9dfeffff8efeffff7ffeffff70feffff61feffff52feffff43feffff34feffff25feffff16feffff07fefffff8fdffffe9fdffffdafdffffcbfdffffbcfdffffadfdffff9efdffff8ffdffff80fdffff71fdffff62fdffff53fdffff44fdffff35fdffff26fdffff17fdffff08fdfffff9fcffffeafcffffdbfcffffccfcffffbdfcffffaefcffff9ffcffff90fcffff81fcffff72fcffff63fcffff54fcffff45fcffff36fcffff27fcffff18fcffff09fcfffffafbffffebfbffffdcfbffffcdfbffffbefbffffaffbffffa0fbffff91fbffff82fbffff73fbffff64fbffff55fbffff46fbffff37fbffff28fbffff19fbffff0afbfffffbfaffffecfaffffddfaffffcefaffffbffaffff00000000b5ffffff6affffff1fffffffd4feffff8afeffff40fefffff6fdffffacfdffff63fdffff1afdffffd1fcffff89fcffff42fcfffffbfbffffb5fbffff70fbffff2bfbffffe7faffffa4faffff62faffff20faffffe0f9ffffa0f9ffff62f9ffff25f9ffffe8f8ffffadf8ffff73f8ffff3bf8ffff03f8ffffcdf7ffff98f7ffff64f7ffff32f7ffff01f7ffffd2f6ffffa4f6ffff78f6ffff4df6ffff24f6fffffcf5ffffd6f5ffffb1f5ffff8ef5ffff6df5ffff4ef5ffff30f5ffff14f5fffffaf4ffffe1f4ffffcaf4ffffb5f4ffffa2f4ffff91f4ffff81f4ffff74f4ffff68f4ffff5ef4ffff56f4ffff50f4ffff4bf4ffff49f4ffff48f4ffff49f4ffff4cf4ffff51f4ffff58f4ffff61f4ffff6cf4ffff78f4ffff86f4ffff96f4ffffa8f4ffffbcf4ffffd2f4ffffe9f4ffff02f5ffff1df5ffff3af5ffff58f5ffff78f5ffff9af5ffffbdf5ffffe2f5ffff09f6ffff31f6ffff5bf6ffff87f6ffffb3f6ffffe2f6ffff12f7ffff43f7ffff76f7ffffaaf7ffffdff7ffff16f8ffff4ef8ffff87f8ffffc1f8fffffdf8ffff39f9ffff77f9ffffb6f9fffff5f9ffff36faffff78faffffbafafffffefaffff42fbffff87fbffffcdfbffff13fcffff5afcffffa2fcffffeafcffff32fdffff7bfdffffc5fdffff0ffeffff59feffffa3feffffeefeffff38ffffff83ffffffceffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900100008d0100008a0100008701000084010000810100007e0100007b0100007801000075010000720100006f0100006c010000690100006601000063010000600100005d0100005a0100005701000054010000510100004e0100004b0100004801000045010000420100003f0100003c010000390100003601000033010000300100002d0100002a0100002701000024010000210100001e0100001b0100001801000015010000120100000f0100000c01000009010000060100000301000000010000fd000000fa000000f7000000f4000000f1000000ee000000eb000000e8000000e5000000e2000000df000000dc000000d9000000d6000000d3000000d0000000cd000000ca000000c7000000c4000000c1000000be000000bb000000b8000000b5000000b2000000af000000ac000000a9000000a6000000a3000000a00000009d0000009a0000009700000094000000910000008e0000008b0000008800000085000000820000007f0000007c000000790000007600000073000000700000006d0000006a0000006700000064000000610000005e0000005b0000005800000055000000520000004f0000004c000000490000004600000043000000400000003d0000003a0000003700000034000000310000002e0000002b0000002800000025000000220000001f0000001c000000190000001600000013000000100000000d0000000a000000070000000400000001000000fefffffffbfffffff8fffffff5fffffff2ffffffefffffffecffffffe9ffffffe6ffffffe3ffffffc2010000c5010000c7010000ca010000cc010000cf010000d1010000d4010000d6010000d9010000db010000de010000e0010000e3010000e5010000e8010000ea010000ed010000ef010000f2010000f4010000f7010000f9010000fc010000fe010000010200000302000006020000080200000b0200000d020000100200001202000015020000170200001a0200000000008000000080000000800000008000000080000000802b0200002e020000300200003302000035020000380200003a0200003d0200003f020000420200004402000047020000490200004c0200004e020000510200005302000056020000580200005b0200005d020000600200006202000065020000670200006a0200006c0200006f020000710200007402000076020000790200007b0200007e020000800200008302000085020000880200008a0200008d0200008f020000920200009402000097020000990200009c0200009e020000a1020000a3020000a6020000a8020000ab020000ad020000b0020000b2020000b5020000b7020000ba020000bc020000bf020000c1020000c4020000c6020000c9020000cb020000ce020000d0020000d3020000d5020000d8020000da020000dd020000df020000e2020000e4020000e7020000e9020000ec020000ee020000f1020000f3020000f6020000f8020000fb020000fd020000000300000203000005030000070300000a0300000c0300000f030000110300001403000016030000190300001b0300001e03000020030000230300002503000028030000" + }, + { + "of": [ + "closed", + "whole", + "append 17:00", + "append 17:05" + ], + "tileId": "5m/1/41477", + "etag": "96002093", + "startMs": 1791806400000, + "partial": true, + "data": "b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff2cf7ffff1df7ffff0ef7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff2cffffff29ffffff26ffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be030000c0030000c3030000c5030000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080" + }, + { + "of": [ + "append 17:00", + "whole" + ], + "tileId": "5m/1/41477", + "etag": "7285c700", + "startMs": 1791806400000, + "partial": true, + "data": "b0faffffa1faffff92faffff83faffff74faffff65faffff56faffff47faffff38faffff29faffff1afaffff0bfafffffcf9ffffedf9ffffdef9ffffcff9ffffc0f9ffffb1f9ffffa2f9ffff93f9ffff84f9ffff75f9ffff66f9ffff57f9ffff48f9ffff39f9ffff2af9ffff1bf9ffff0cf9fffffdf8ffffeef8ffffdff8ffffd0f8ffffc1f8ffffb2f8ffffa3f8ffff94f8ffff85f8ffff76f8ffff67f8ffff58f8ffff49f8ffff3af8ffff2bf8ffff1cf8ffff0df8fffffef7ffffeff7ffffe0f7ffffd1f7ffffc2f7ffffb3f7ffffa4f7ffff95f7ffff86f7ffff77f7ffff68f7ffff59f7ffff4af7ffff3bf7ffff000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080bcffffffb9ffffffb6ffffffb3ffffffb0ffffffadffffffaaffffffa7ffffffa4ffffffa1ffffff9effffff9bffffff98ffffff95ffffff92ffffff8fffffff8cffffff89ffffff86ffffff83ffffff80ffffff7dffffff7affffff77ffffff74ffffff71ffffff6effffff6bffffff68ffffff65ffffff62ffffff5fffffff5cffffff59ffffff56ffffff53ffffff50ffffff4dffffff4affffff47ffffff44ffffff41ffffff3effffff3bffffff38ffffff35ffffff32ffffff2fffffff0000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000802a0300002d0300002f030000320300003403000037030000390300003c0300003e030000410300004303000046030000480300004b0300004d030000500300005203000055030000570300005a0300005c0300005f030000610300006403000066030000690300006b0300006e030000700300007303000075030000780300007a0300007d0300007f030000820300008403000087030000890300008c0300008e030000910300009303000096030000980300009b0300009d030000a0030000a2030000a5030000a7030000aa030000ac030000af030000b1030000b4030000b6030000b9030000bb030000be030000000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080000000800000008000000080" + } + ], + "tilesGet": { + "body": "a16468617665a16a356d2f312f3431343736683938366531386364", + "envelope": "a36162581ba16468617665a16a356d2f312f343134373668393836653138636461746974696c65732e6765746373696740" + }, + "tilesEnd": { + "body": "a1616e03", + "envelope": "a3616244a1616e0361746974696c65732e656e646373696740" + }, + "notes": { + "forged": "The last append with its data changed after signing: it must not verify, and is never held or laid.", + "have": "The tiles.get have that holds the closed tile: the service answers it with the pieces alone.", + "laid": "What Lay makes of the named tiles, in that order. A trailing tile stays partial; its etag is that of the data as laid.", + "tiles": "One tile of each shape, in the order the box sends them, signed by boxKey. Each passes Tiles.Check." + }, + "have": { + "answer": [ + "whole", + "append 17:00", + "append 17:05" + ] + } +} diff --git a/go/protocol/cloudwire/tiles.go b/go/protocol/cloudwire/tiles.go new file mode 100644 index 000000000..869072ad6 --- /dev/null +++ b/go/protocol/cloudwire/tiles.go @@ -0,0 +1,231 @@ +package cloudwire + +import ( + "errors" + "fmt" + "slices" + "sort" + + "github.com/srcfl/ftw/go/protocol" +) + +// Tiles: the last 48 hours of the box's 5-minute history, so a phone can draw +// it while the box cannot answer (docs/protocol-cloud.md, tiles). +// +// A tiles body is the app protocol's hist.chunk with the common header. It +// comes in three shapes, all at 5 minutes and stride 1: +// +// - closed: partial false, startMs the tile's start, every point. The same +// tile id, etag and bytes as the box serves over Noise for TileSeries. +// - whole: partial true, startMs the tile's start, every point, missing +// from the first bucket that has not closed. +// - append: partial true, startMs a later bucket, only the buckets that +// closed since the last piece of that tile. +// +// A whole tile and an append are pieces of the trailing tile. Lay puts the +// pieces a receiver holds back together. + +// MsgTilesGet asks the service for the tiles a phone lacks, as TilesGet. The +// service answers with the box's tiles envelopes, unchanged, then MsgTilesEnd. +const MsgTilesGet = "tiles.get" + +// MsgTilesEnd closes the service's answer to tiles.get. It is the service's +// own and says only how many envelopes went before it. +const MsgTilesEnd = "tiles.end" + +// The geometry FTW Cloud keeps. +const ( + // TileRes is the one resolution the box sends. + TileRes = protocol.ResN5m + // TileKeepMs is how far back tiles are kept: a tile that ended earlier + // than this before now is gone. + TileKeepMs = 48 * 3_600_000 + // TileSettleMs is how long after a bucket ends the box counts it closed. + // The history writer commits every 15 s. + TileSettleMs = 30_000 + // TileAppendsMax is how many appends follow a whole piece of the + // trailing tile before the box sends it whole again: once an hour. + TileAppendsMax = 11 + // TilePiecesMax is how many pieces of the trailing tile the service + // holds. A whole piece replaces the ones before it. + TilePiecesMax = 16 + // TilesClosedMax is how many closed tiles 48 hours can touch. + TilesClosedMax = 5 + // TileSeriesMax caps the columns of one tile, as hist.query does. + TileSeriesMax = 16 +) + +// TileSeries are the columns the box sends, in this order. They are what the +// app asks for over Noise, so a closed tile is the same bytes either way. +var TileSeries = []string{"grid_w", "pv_w", "battery_w", "load_w"} + +// Tiles is one tile, or one piece of the trailing tile. +type Tiles struct { + Header + TileID string `cbor:"tileId"` + Etag string `cbor:"etag"` + Res protocol.Resolution `cbor:"res"` + StartMs int64 `cbor:"startMs"` + StepMs int64 `cbor:"stepMs"` + Series []string `cbor:"series"` + // Data is int32 LE, one block per series, protocol.MissingSample where + // the box has no record. + Data []byte `cbor:"data"` + Partial bool `cbor:"partial"` +} + +// TilesGet asks for tiles. Have maps tile id to etag for the closed tiles the +// phone holds; the service sends every closed tile it holds that is not +// there with that etag, and every piece of the trailing tile. +type TilesGet struct { + Have map[string]string `cbor:"have"` +} + +// TilesEnd closes the answer to tiles.get. +type TilesEnd struct { + // N is how many tiles envelopes went before it. + N int `cbor:"n"` +} + +// Chunk is the tile as the app protocol carries it, without the header. +func (t Tiles) Chunk() protocol.HistChunk { + return protocol.HistChunk{TileID: t.TileID, Etag: t.Etag, Res: t.Res, StartMs: t.StartMs, + StepMs: t.StepMs, Series: slices.Clone(t.Series), Data: slices.Clone(t.Data), Partial: t.Partial} +} + +// TileStart is the start of the tile this body belongs to. +func (t Tiles) TileStart() int64 { return protocol.TileStartFor(t.Res, t.StartMs) } + +// TileEnd is where that tile ends. +func (t Tiles) TileEnd() int64 { return t.TileStart() + protocol.ResolutionTileSpanMs[t.Res] } + +// Points is how many points each series carries. +func (t Tiles) Points() int { + if len(t.Series) == 0 { + return 0 + } + return len(t.Data) / 4 / len(t.Series) +} + +// Whole says the body starts at its tile's start: a closed tile, or a whole +// piece that replaces every piece of that tile before it. +func (t Tiles) Whole() bool { return t.StartMs == t.TileStart() } + +// ErrTile is a tiles body that breaks the shapes above. +var ErrTile = errors.New("cloudwire: not a tile") + +// Check says whether t is one of the three shapes. The service holds no +// other; the phone lays no other. +func (t Tiles) Check() error { + bad := func(why string) error { return fmt.Errorf("%w: %s", ErrTile, why) } + if t.Res != TileRes || t.StepMs != protocol.ResolutionStepMs[TileRes] { + return bad("not 5-minute steps") + } + if t.StartMs < 0 || t.StartMs%t.StepMs != 0 { + return bad("start off the 5-minute grid") + } + if t.TileID != protocol.TileIDFor(t.Res, 1, t.TileStart()) { + return bad("tile id does not name the tile its start is in") + } + if len(t.Series) == 0 || len(t.Series) > TileSeriesMax { + return bad("series count") + } + seen := map[string]bool{} + for _, s := range t.Series { + if s == "" || len(s) > 64 || seen[s] { + return bad("series names") + } + seen[s] = true + } + n := t.Points() + if n < 1 || len(t.Data) != n*4*len(t.Series) { + return bad("data is not whole columns") + } + first := int((t.StartMs - t.TileStart()) / t.StepMs) + if first+n > int(protocol.PointsPerTile(t.Res)) { + return bad("data runs past the tile") + } + if !t.Partial && (!t.Whole() || n != int(protocol.PointsPerTile(t.Res))) { + return bad("a closed tile that is not the whole tile") + } + if t.Etag != protocol.EtagOf(t.Data) { + return bad("etag") + } + return nil +} + +// Piece cuts points [from, to) of each series out of a packed tile, as the +// box sends an append or a whole piece. from 0 starts at the tile's start. +// The header is the caller's. +func Piece(c protocol.HistChunk, from, to int) Tiles { + per := 0 + if len(c.Series) > 0 { + per = len(c.Data) / 4 / len(c.Series) + } + to = min(to, per) + from = min(max(from, 0), to) + data := make([]byte, 0, len(c.Series)*(to-from)*4) + for s := range c.Series { + base := s * per * 4 + data = append(data, c.Data[base+from*4:base+to*4]...) + } + return Tiles{TileID: c.TileID, Etag: protocol.EtagOf(data), Res: c.Res, + StartMs: c.StartMs + int64(from)*c.StepMs, StepMs: c.StepMs, + Series: slices.Clone(c.Series), Data: data, Partial: true} +} + +// Lay puts tiles back together, in the order the box sent them: a closed tile +// as it is; the trailing tile from its newest whole piece, with each later +// append laid over it. A piece of a tile already closed changes nothing, and +// neither does a body that fails Check. The result has one chunk per tile, +// oldest first, each with every point; a trailing tile stays partial, its +// etag that of the data as laid, never one to offer as held. +func Lay(bodies []Tiles) []protocol.HistChunk { + byID := map[string]*protocol.HistChunk{} + for _, t := range bodies { + if t.Check() != nil { + continue + } + held := byID[t.TileID] + if held != nil && !held.Partial { + continue + } + if !t.Partial { + c := t.Chunk() + byID[t.TileID] = &c + continue + } + per := int(protocol.PointsPerTile(t.Res)) + if held == nil || t.Whole() || !slices.Equal(held.Series, t.Series) { + c := protocol.HistChunk{TileID: t.TileID, Res: t.Res, StartMs: t.TileStart(), StepMs: t.StepMs, + Series: slices.Clone(t.Series), Data: missing(len(t.Series) * per), Partial: true} + held = &c + byID[t.TileID] = held + } + n := t.Points() + at := int((t.StartMs - t.TileStart()) / t.StepMs) + for s := range t.Series { + copy(held.Data[(s*per+at)*4:(s*per+at+n)*4], t.Data[s*n*4:(s+1)*n*4]) + } + } + out := make([]protocol.HistChunk, 0, len(byID)) + for _, c := range byID { + if c.Partial { + c.Etag = protocol.EtagOf(c.Data) + } + out = append(out, *c) + } + sort.Slice(out, func(a, b int) bool { return out[a].StartMs < out[b].StartMs }) + return out +} + +// missing is n samples of protocol.MissingSample. +func missing(n int) []byte { + b := make([]byte, n*4) + m := int32(protocol.MissingSample) + u := uint32(m) + for i := 0; i < n; i++ { + b[i*4], b[i*4+1], b[i*4+2], b[i*4+3] = byte(u), byte(u>>8), byte(u>>16), byte(u>>24) + } + return b +} diff --git a/go/protocol/cloudwire/tiles_test.go b/go/protocol/cloudwire/tiles_test.go new file mode 100644 index 000000000..18fb4eae2 --- /dev/null +++ b/go/protocol/cloudwire/tiles_test.go @@ -0,0 +1,389 @@ +package cloudwire + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "math" + "os" + "slices" + "testing" + + "github.com/srcfl/ftw/go/protocol" +) + +// testdata/cloud-tile-vectors.json freezes one tile of each shape, signed by +// the vector box key of cloud-vectors.json, how three pieces lay back +// together, and the bodies of tiles.get and tiles.end. srcfl/ftw-cloud checks +// and holds them; the phone verifies, decodes and lays them. +// +// go test ./cloudwire -run TestTileVectors -update + +const tileVectorsPath = "testdata/cloud-tile-vectors.json" + +// The vectors' tiles: the tile before 2026-10-12 12:00 UTC, closed, and the +// one after it, still filling at 17:15. +const ( + vectorTileStart = int64(1_791_806_400_000) // 2026-10-12 12:00 UTC + vectorTilePrev = vectorTileStart - 43_200_000 +) + +// vectorWindows is the ledger the vectors pack from: a value for every +// bucket of both tiles before 17:15, but none for the house between 03:00 +// and 03:30, and no battery for the first hour of the afternoon. +func vectorWindows() map[string][]protocol.HistorySample { + out := map[string][]protocol.HistorySample{} + for at := vectorTilePrev; at < vectorTileStart+5*3_600_000+15*60_000; at += 300_000 { + i := float64((at - vectorTilePrev) / 300_000) + out["grid_w"] = append(out["grid_w"], protocol.HistorySample{StartMs: at, W: 800 - 15*i}) + out["pv_w"] = append(out["pv_w"], protocol.HistorySample{StartMs: at, W: -math.Max(0, 3000*math.Sin(i/40))}) + if at < vectorTileStart || at >= vectorTileStart+3_600_000 { + out["battery_w"] = append(out["battery_w"], protocol.HistorySample{StartMs: at, W: 400 - 3*i}) + } + if at < vectorTilePrev+3*3_600_000 || at >= vectorTilePrev+3*3_600_000+30*60_000 { + out["load_w"] = append(out["load_w"], protocol.HistorySample{StartMs: at, W: 450 + 2.5*i}) + } + } + return out +} + +// vectorPack packs a tile as the box does: buckets from closedThrough on are +// not there yet. +func vectorPack(start, closedThrough int64) protocol.HistChunk { + windows := map[string][]protocol.HistorySample{} + for name, samples := range vectorWindows() { + for _, s := range samples { + if s.StartMs < closedThrough { + windows[name] = append(windows[name], s) + } + } + } + return protocol.PackTile(TileRes, 1, start, TileSeries, windows, closedThrough) +} + +func header(seq uint64, wall int64) Header { + return Header{Seq: seq, Boot: bytes.Repeat([]byte{0xb0}, BootBytes), UptimeMs: 60_000 + int64(seq)*10, WallMs: i64(wall)} +} + +// The vectors' bodies, in the order the box sends them. +func vectorTiles() []struct { + name string + body Tiles +} { + closed := vectorPack(vectorTilePrev, vectorTileStart+30_000) + whole := Piece(vectorPack(vectorTileStart, vectorTileStart+5*3_600_000), 0, 144) + first := Piece(vectorPack(vectorTileStart, vectorTileStart+5*3_600_000+5*60_000), 60, 61) + second := Piece(vectorPack(vectorTileStart, vectorTileStart+5*3_600_000+15*60_000), 61, 63) + closedBody := Tiles{TileID: closed.TileID, Etag: closed.Etag, Res: closed.Res, StartMs: closed.StartMs, + StepMs: closed.StepMs, Series: closed.Series, Data: closed.Data, Partial: closed.Partial} + + wall := vectorTileStart + 5*3_600_000 + closedBody.Header = header(20, vectorTileStart+30_000) + whole.Header = header(21, wall+30_000) + first.Header = header(22, wall+5*60_000+30_000) + second.Header = header(23, wall+15*60_000+30_000) + return []struct { + name string + body Tiles + }{{"closed", closedBody}, {"whole", whole}, {"append 17:00", first}, {"append 17:05", second}} +} + +func TestTheThreeShapesCheck(t *testing.T) { + for _, v := range vectorTiles() { + if err := v.body.Check(); err != nil { + t.Errorf("%s: %v", v.name, err) + } + } + closed := vectorTiles()[0].body + if closed.Partial || !closed.Whole() || closed.Points() != 144 { + t.Fatalf("closed tile: partial %v whole %v points %d", closed.Partial, closed.Whole(), closed.Points()) + } + if a := vectorTiles()[3].body; a.Whole() || a.Points() != 2 || a.StartMs != vectorTileStart+5*3_600_000+5*60_000 { + t.Fatalf("append: whole %v points %d start %d", a.Whole(), a.Points(), a.StartMs) + } +} + +// A closed tile is what the box serves over Noise for the same series: the +// same id, etag and bytes. +func TestAClosedTileIsTheNoiseTile(t *testing.T) { + noise := protocol.PackTile(TileRes, 1, vectorTilePrev, TileSeries, vectorWindows(), vectorTileStart+3_600_000) + got := vectorTiles()[0].body.Chunk() + if got.TileID != noise.TileID || got.Etag != noise.Etag || !bytes.Equal(got.Data, noise.Data) || got.Partial != noise.Partial { + t.Fatalf("closed tile %s %s differs from Noise's %s %s", got.TileID, got.Etag, noise.TileID, noise.Etag) + } +} + +func TestCheckRefusesWhatIsNotATile(t *testing.T) { + good := vectorTiles()[2].body + cases := map[string]func(*Tiles){ + "etag": func(b *Tiles) { b.Etag = "00000000" }, + "another tile's id": func(b *Tiles) { b.TileID = protocol.TileIDFor(TileRes, 1, vectorTilePrev) }, + "stride 2 id": func(b *Tiles) { b.TileID = protocol.TileIDFor(TileRes, 2, vectorTileStart) }, + "hourly": func(b *Tiles) { b.Res = protocol.ResN1h }, + "step": func(b *Tiles) { b.StepMs = 600_000 }, + "off the grid": func(b *Tiles) { b.StartMs++ }, + "half a column": func(b *Tiles) { + b.Data = b.Data[:len(b.Data)-2] + b.Etag = protocol.EtagOf(b.Data) + }, + "no series": func(b *Tiles) { b.Series = nil }, + "twice the same series": func(b *Tiles) { + b.Series = []string{"grid_w", "grid_w", "pv_w", "load_w"} + }, + "past the tile": func(b *Tiles) { + b.StartMs = vectorTileStart + 143*300_000 + b.Data = append(b.Data, b.Data...) + b.Etag = protocol.EtagOf(b.Data) + }, + "closed but a piece": func(b *Tiles) { b.Partial = false }, + } + for name, mutate := range cases { + b := good + b.Series = slices.Clone(good.Series) + b.Data = slices.Clone(good.Data) + mutate(&b) + if err := b.Check(); !errors.Is(err, ErrTile) { + t.Errorf("%s: Check() = %v, want ErrTile", name, err) + } + } +} + +func sample(c protocol.HistChunk, series string, point int) int32 { + s := slices.Index(c.Series, series) + per := len(c.Data) / 4 / len(c.Series) + return int32(binary.LittleEndian.Uint32(c.Data[(s*per+point)*4:])) +} + +// The whole piece and two appends lay back into the tile the box would pack +// at 17:15, less the bucket that has not closed. +func TestLayPutsThePiecesBackTogether(t *testing.T) { + v := vectorTiles() + laid := Lay([]Tiles{v[0].body, v[1].body, v[2].body, v[3].body}) + if len(laid) != 2 || laid[0].TileID != v[0].body.TileID || laid[1].TileID != v[1].body.TileID { + t.Fatalf("laid %d tiles", len(laid)) + } + want := vectorPack(vectorTileStart, vectorTileStart+5*3_600_000+15*60_000) + got := laid[1] + if !got.Partial || got.StartMs != vectorTileStart || !bytes.Equal(got.Data, want.Data) || got.Etag != protocol.EtagOf(want.Data) { + t.Fatalf("trailing tile does not match the box's own pack") + } + if sample(got, "battery_w", 3) != protocol.MissingSample || sample(got, "grid_w", 63) != protocol.MissingSample { + t.Fatal("a hole, or a bucket not closed yet, reads as a value") + } + if laid[0].Partial || laid[0].Etag != v[0].body.Etag { + t.Fatal("the closed tile changed") + } +} + +func TestLayKeepsTheNewestWholeAndTheClosedTile(t *testing.T) { + v := vectorTiles() + // An append, then a whole piece sent later: the whole one wins, and so + // the append before it is gone. + laid := Lay([]Tiles{v[2].body, v[1].body}) + if sample(laid[0], "grid_w", 60) != protocol.MissingSample { + t.Fatal("an append before a whole piece survived it") + } + // A piece of a tile already closed changes nothing. + piece := Piece(vectorPack(vectorTilePrev, vectorTilePrev+3_600_000), 0, 144) + laid = Lay([]Tiles{v[0].body, piece}) + if laid[0].Partial || laid[0].Etag != v[0].body.Etag { + t.Fatal("a piece reopened a closed tile") + } + // A body that fails Check is skipped. + bad := v[3].body + bad.Etag = "00000000" + laid = Lay([]Tiles{v[1].body, v[2].body, bad}) + if sample(laid[0], "grid_w", 61) != protocol.MissingSample { + t.Fatal("a body with a wrong etag was laid") + } +} + +type tileVectorFile struct { + BoxKey vectorKey `json:"boxKey"` + Series []string `json:"series"` + Tiles []tileVector `json:"tiles"` + Forged tileVector `json:"forged"` + Laid []tileVectorLaid `json:"laid"` + Get tileVectorExchange `json:"tilesGet"` + End tileVectorExchange `json:"tilesEnd"` + Notes map[string]string `json:"notes"` + Have map[string][]string `json:"have"` +} + +type tileVector struct { + Name string `json:"name"` + TileID string `json:"tileId"` + Etag string `json:"etag"` + StartMs int64 `json:"startMs"` + Points int `json:"points"` + Partial bool `json:"partial"` + Seq uint64 `json:"seq"` + Body string `json:"body"` + SignedInputSHA256 string `json:"signedInputSha256"` + Sig string `json:"sig"` + Envelope string `json:"envelope"` + Frame string `json:"frame"` +} + +type tileVectorLaid struct { + Of []string `json:"of"` + TileID string `json:"tileId"` + Etag string `json:"etag"` + StartMs int64 `json:"startMs"` + Partial bool `json:"partial"` + Data string `json:"data"` +} + +type tileVectorExchange struct { + Body string `json:"body"` + Envelope string `json:"envelope"` +} + +func TestTileVectors(t *testing.T) { + box := testKey(t, "ftw.cloud vector box") + home, boxPK := homeOf(t, box) + + var stored tileVectorFile + raw, err := os.ReadFile(tileVectorsPath) + switch { + case err == nil: + if err := json.Unmarshal(raw, &stored); err != nil { + t.Fatal(err) + } + case !*update: + t.Fatalf("%v (run with -update to create it)", err) + } + + out := tileVectorFile{ + BoxKey: vectorKey{D: hex.EncodeToString(box.D.FillBytes(make([]byte, 32))), PK: hex.EncodeToString(boxPK), Home: home}, + Series: TileSeries, + Notes: map[string]string{ + "tiles": "One tile of each shape, in the order the box sends them, signed by boxKey. Each passes Tiles.Check.", + "forged": "The last append with its data changed after signing: it must not verify, and is never held or laid.", + "laid": "What Lay makes of the named tiles, in that order. A trailing tile stays partial; its etag is that of the data as laid.", + "have": "The tiles.get have that holds the closed tile: the service answers it with the pieces alone.", + }, + } + signed := func(i int, name string, b Tiles) tileVector { + body, err := Marshal(b) + if err != nil { + t.Fatal(err) + } + input, err := SignedInput(home, MsgTiles, body) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(input) + v := tileVector{Name: name, TileID: b.TileID, Etag: b.Etag, StartMs: b.StartMs, Points: b.Points(), Partial: b.Partial, + Seq: b.Seq, Body: hex.EncodeToString(body), SignedInputSHA256: hex.EncodeToString(digest[:])} + var sig []byte + if i < len(stored.Tiles) && stored.Tiles[i].Name == name && stored.Tiles[i].SignedInputSHA256 == v.SignedInputSHA256 { + sig, _ = hex.DecodeString(stored.Tiles[i].Sig) + } else if *update { + if sig, err = Sign(rand.Reader, box, home, MsgTiles, body); err != nil { + t.Fatal(err) + } + } else { + t.Fatalf("tile vector %d is not %q as stored; regenerate with -update", i, name) + } + env := Envelope{T: MsgTiles, B: body, Sig: sig} + if err := Verify(&box.PublicKey, home, env); err != nil { + t.Fatalf("%s: stored signature does not verify: %v", name, err) + } + v.Sig = hex.EncodeToString(sig) + envBytes, err := env.Encode() + if err != nil { + t.Fatal(err) + } + frame, err := EncodeFrame(envBytes) + if err != nil { + t.Fatal(err) + } + v.Envelope, v.Frame = hex.EncodeToString(envBytes), hex.EncodeToString(frame) + return v + } + + tiles := vectorTiles() + var bodies []Tiles + for i, v := range tiles { + out.Tiles = append(out.Tiles, signed(i, v.name, v.body)) + bodies = append(bodies, v.body) + } + + // The forged tile: the last append's signature over other data. + last := out.Tiles[len(out.Tiles)-1] + forged := tiles[len(tiles)-1].body + forged.Data = slices.Clone(forged.Data) + forged.Data[0] ^= 1 + forged.Etag = protocol.EtagOf(forged.Data) + fb, err := Marshal(forged) + if err != nil { + t.Fatal(err) + } + sig, _ := hex.DecodeString(last.Sig) + fenv := Envelope{T: MsgTiles, B: fb, Sig: sig} + if Verify(&box.PublicKey, home, fenv) == nil { + t.Fatal("the forged tile verifies") + } + fraw, err := fenv.Encode() + if err != nil { + t.Fatal(err) + } + fframe, err := EncodeFrame(fraw) + if err != nil { + t.Fatal(err) + } + out.Forged = tileVector{Name: "forged append", TileID: forged.TileID, Etag: forged.Etag, StartMs: forged.StartMs, + Points: forged.Points(), Partial: true, Seq: forged.Seq, Body: hex.EncodeToString(fb), Sig: last.Sig, + Envelope: hex.EncodeToString(fraw), Frame: hex.EncodeToString(fframe)} + + for _, of := range [][]int{{0}, {1}, {1, 2, 3}, {0, 1, 2, 3}, {2, 1}} { + var in []Tiles + var names []string + for _, i := range of { + in = append(in, bodies[i]) + names = append(names, tiles[i].name) + } + for _, c := range Lay(in) { + out.Laid = append(out.Laid, tileVectorLaid{Of: names, TileID: c.TileID, Etag: c.Etag, StartMs: c.StartMs, + Partial: c.Partial, Data: hex.EncodeToString(c.Data)}) + } + } + + have := map[string]string{bodies[0].TileID: bodies[0].Etag} + out.Have = map[string][]string{"answer": {"whole", "append 17:00", "append 17:05"}} + for _, ex := range []struct { + dst *tileVectorExchange + t string + body any + }{{&out.Get, MsgTilesGet, TilesGet{Have: have}}, {&out.End, MsgTilesEnd, TilesEnd{N: 3}}} { + body, err := Marshal(ex.body) + if err != nil { + t.Fatal(err) + } + env, err := Unsigned(ex.t, ex.body) + if err != nil { + t.Fatal(err) + } + *ex.dst = tileVectorExchange{Body: hex.EncodeToString(body), Envelope: hex.EncodeToString(env)} + } + + got, err := json.MarshalIndent(out, "", " ") + if err != nil { + t.Fatal(err) + } + got = append(got, '\n') + if *update { + if err := os.WriteFile(tileVectorsPath, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + if !bytes.Equal(got, raw) { + t.Fatalf("%s is out of date; regenerate with -update", tileVectorsPath) + } +} From b8cffc57446a7e2415948d795b77a3bee13967ed Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 23 Oct 2026 16:03:14 +0200 Subject: [PATCH 2/5] feat(cloudlink): send FTW Cloud 48 hours of history tiles With ftw.history.read in the owner's grant and NTP time, the box sends the trailing 5-minute tile as its buckets close (an append, whole once an hour), each tile as it closes, and at connect or a resync the tiles of the last 48 hours that hello_ok's have lacks. Closed tiles are packed from the same ledger read as the app's Noise history, so they are the same bytes. Tiles go last in a tick and count against the byte budget. Co-Authored-By: Claude Opus 5.5 --- .changeset/cloud-history-tiles.md | 5 + docs/adr/0008-ftw-cloud.md | 1 + go/cmd/ftw/cloud_link.go | 15 +- go/internal/cloudlink/conn.go | 3 + go/internal/cloudlink/link.go | 5 +- go/internal/cloudlink/link_test.go | 10 +- go/internal/cloudlink/sender.go | 11 +- go/internal/cloudlink/tiles.go | 207 ++++++++++++++ go/internal/cloudlink/tiles_test.go | 410 ++++++++++++++++++++++++++++ 9 files changed, 661 insertions(+), 6 deletions(-) create mode 100644 .changeset/cloud-history-tiles.md create mode 100644 go/internal/cloudlink/tiles.go create mode 100644 go/internal/cloudlink/tiles_test.go diff --git a/.changeset/cloud-history-tiles.md b/.changeset/cloud-history-tiles.md new file mode 100644 index 000000000..7c566cc3d --- /dev/null +++ b/.changeset/cloud-history-tiles.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +With FTW Cloud on and history in the owner's grant, the box sends FTW Cloud its last 48 hours of 5-minute history, so the iPhone app can draw a node's history while the box is offline. Each closed 12-hour tile is the same bytes the app gets from the box directly. The box adds each 5-minute bucket 30 seconds after it ends, and sends the tiles FTW Cloud lacks when it reconnects. It sends no history without NTP time, and none past the daily byte budget. The extra traffic is about 100 KB a day. diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index 8ad971d96..a5c14502d 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -212,6 +212,7 @@ The privacy notice says so. | Store | Kept | |---|---| | Live state | Memory only; readings dropped an hour after the box goes quiet | +| History tiles | Memory only; each 12-hour tile of 5-minute history until 48 hours after it ends, box online or not | | Registry | Until purge, or 90 days without a roster | | APNs registrations | Until purge, or 30 days without the box | | Events | A one-hour ring in memory | diff --git a/go/cmd/ftw/cloud_link.go b/go/cmd/ftw/cloud_link.go index d9a133573..8f2fedaf1 100644 --- a/go/cmd/ftw/cloud_link.go +++ b/go/cmd/ftw/cloud_link.go @@ -50,6 +50,9 @@ type cloudSite struct { // With it, the link sends the car class, and a charger's level and // goal fields are views of its session's car. cars *cars.Runtime + // history is the energy ledger, read as the app's history over Noise + // reads it, so a closed tile is the same bytes either way. + history *appHistory sessMu sync.Mutex sessions map[string]*plugSession @@ -63,6 +66,16 @@ type plugSession struct { n int } +var _ cloudlink.HistorySite = (*cloudSite)(nil) + +// HistoryWindow serves the tiles class. +func (c *cloudSite) HistoryWindow(ctx context.Context, names []string, stepMs, fromMs, toMs int64) (map[string][]protocol.HistorySample, error) { + if c.history == nil { + return nil, errors.New("no energy ledger") + } + return c.history.Window(ctx, names, stepMs, fromMs, toMs) +} + func (c *cloudSite) Statistics(ctx context.Context, now time.Time) (cloudwire.Stats, bool) { if srv := c.api.srv.Load(); srv != nil { return srv.CloudStatistics(ctx, now) @@ -394,7 +407,7 @@ func startCloudLink( if st == nil || enroll == nil { return nil } - site := &cloudSite{api: gateway, cars: carsRT, + site := &cloudSite{api: gateway, cars: carsRT, history: &appHistory{st: st}, site: newAppSite(tel, ctrl, ctrlMu, revision, siteMeterStale, measurementOptions)} if priceSvc != nil && priceSvc.Zone != "" && priceSvc.Store != nil { site.prices = &appPrices{svc: priceSvc} diff --git a/go/internal/cloudlink/conn.go b/go/internal/cloudlink/conn.go index c31f29d82..3e240d65b 100644 --- a/go/internal/cloudlink/conn.go +++ b/go/internal/cloudlink/conn.go @@ -300,6 +300,7 @@ func (l *Link) serve(ctx context.Context, conn uplinkConn, snap grantSnapshot) e // so revokes made since then still arrive. l.bumpRoster(ok.RosterSeq) } + w.have = ok.Have tunOut, tunBulk := make(chan []byte, tunQueue), make(chan []byte, tunBulkQueue) l.mu.Lock() @@ -454,6 +455,8 @@ type writer struct { conn uplinkConn link *Link mu sync.Mutex + // have is the closed tiles hello_ok said the service holds. + have map[string]string } // send seals body as type t, frames it and writes it. The byte budget is diff --git a/go/internal/cloudlink/link.go b/go/internal/cloudlink/link.go index c0b19a624..9593a259b 100644 --- a/go/internal/cloudlink/link.go +++ b/go/internal/cloudlink/link.go @@ -1196,8 +1196,9 @@ func (snap grantSnapshot) allowed(class string) bool { func (l *Link) caps(snap grantSnapshot) []string { var out []string _, cars := l.opts.Site.(CarSite) - for _, c := range []string{cloudwire.MsgLive, cloudwire.MsgLP, cloudwire.MsgCar, cloudwire.MsgPrices, cloudwire.MsgHB, cloudwire.MsgEvent} { - if snap.allowed(c) && (c != cloudwire.MsgCar || cars) { + _, history := l.opts.Site.(HistorySite) + for _, c := range []string{cloudwire.MsgLive, cloudwire.MsgLP, cloudwire.MsgCar, cloudwire.MsgPrices, cloudwire.MsgHB, cloudwire.MsgEvent, cloudwire.MsgTiles} { + if snap.allowed(c) && (c != cloudwire.MsgCar || cars) && (c != cloudwire.MsgTiles || history) { out = append(out, c) } } diff --git a/go/internal/cloudlink/link_test.go b/go/internal/cloudlink/link_test.go index bcc5f51f8..e56080d5b 100644 --- a/go/internal/cloudlink/link_test.go +++ b/go/internal/cloudlink/link_test.go @@ -217,6 +217,8 @@ type fakeService struct { // dials again. kick int statsPolicy string + // have is the closed tiles hello_ok names as held. + have map[string]string // usedEarly says, per QUIC connection, whether it took early data. usedEarly []bool } @@ -326,8 +328,14 @@ func (s *fakeService) serve(conn uplinkConn, ekm []byte) { if s.wrongKey { signer, _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) } + have := map[string]string{} + s.mu.Lock() + for id, etag := range s.have { + have[id] = etag + } + s.mu.Unlock() s.write(conn, signer, home, cloudwire.MsgHelloOK, cloudwire.HelloOK{ - BoxNonce: hello.BoxNonce, EKM: okEKM, Have: map[string]string{}, RosterSeq: s.rosterSeqOK, StatsPolicy: s.statsPolicy}) + BoxNonce: hello.BoxNonce, EKM: okEKM, Have: have, RosterSeq: s.rosterSeqOK, StatsPolicy: s.statsPolicy}) if s.afterHello != nil { go s.afterHello(conn, home) } diff --git a/go/internal/cloudlink/sender.go b/go/internal/cloudlink/sender.go index b377eaf7a..9c4f3212a 100644 --- a/go/internal/cloudlink/sender.go +++ b/go/internal/cloudlink/sender.go @@ -101,6 +101,9 @@ type cadence struct { // arrived says a driver finished a poll since live was last composed, // while a phone watches. arrived bool + + // tiles is what this connection sent of the tiles class. + tiles tileState } // liveCheckEvery is how often the source table is looked at, so a table @@ -115,7 +118,7 @@ func (w *writer) run(ctx context.Context, events <-chan cloudwire.Envelope) erro connHome := l.current().home // A roster goes at every connect, which also renews its ttlS: the // service counts that from when it received the last one. - st := &cadence{full: true} + st := &cadence{full: true, tiles: newTileState(w.have)} var purgeSince time.Time // slot fires when an arrival waits for the watched live's period. var slot <-chan time.Time @@ -208,6 +211,9 @@ func (w *writer) downlink(snap grantSnapshot, st *cadence, env cloudwire.Envelop if slices.Contains(b.What, cloudwire.ResyncRoster) { st.rosterAt = time.Time{} } + if slices.Contains(b.What, cloudwire.ResyncTiles) { + st.tiles.resync(b.Tiles) + } } return nil } @@ -385,7 +391,8 @@ func (w *writer) tick(snap grantSnapshot, st *cadence, now time.Time) error { st.anyAt = now } } - return nil + // Tiles go last: a reading never waits for the ledger. + return w.tiles(snap, st, now) } // takeRoster says whether a roster is due, and builds it from the link as it diff --git a/go/internal/cloudlink/tiles.go b/go/internal/cloudlink/tiles.go new file mode 100644 index 000000000..d7ef8b69b --- /dev/null +++ b/go/internal/cloudlink/tiles.go @@ -0,0 +1,207 @@ +package cloudlink + +import ( + "context" + "slices" + "time" + + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// HistorySite is a Site that keeps history. HistoryWindow answers stored +// series as the app protocol's history provider does: every series that +// overlaps [fromMs, toMs), at the stored width stepMs, without missing +// buckets. A Site without it sends no tiles. +type HistorySite interface { + HistoryWindow(ctx context.Context, names []string, stepMs, fromMs, toMs int64) (map[string][]protocol.HistorySample, error) +} + +// tilesRecheckEvery is how often the closed tiles of the window are packed +// again, so a gap the ledger repaired reaches the service. A test shortens +// it. +var tilesRecheckEvery = time.Hour + +const ( + // tilesRetryAfter holds tiles back after a read failed or the budget + // refused one. + tilesRetryAfter = time.Minute + // tilesReadTimeout bounds one read of the ledger. + tilesReadTimeout = 5 * time.Second + // tilesNamedMax is how many tiles a resync may name. + tilesNamedMax = cloudwire.TilesClosedMax + 1 +) + +// tileState is one connection's view of the tiles it has sent. +type tileState struct { + // have is each closed tile the service holds, by id, as hello_ok named + // them and as sent since. + have map[string]string + // trail is the start of the trailing tile and through the end of the + // last of its buckets sent on this connection; through == trail means + // none of it has gone. appends counts appends since it last went whole. + trail, through int64 + appends int + // all asks for every tile of the window, at connect and after a resync + // that names none; force sends them whatever have says. named are the + // tiles a resync named. + all, force bool + named []string + recheckAt time.Time + retryAt time.Time +} + +func newTileState(have map[string]string) tileState { + held := make(map[string]string, len(have)) + for id, etag := range have { + held[id] = etag + } + return tileState{have: held, all: true} +} + +// resync takes the service's resync of tiles: the tiles it names, or every +// one of the window when it names none. +func (ts *tileState) resync(named []string) { + ts.force = true + if len(named) == 0 || len(named) > tilesNamedMax { + ts.all, ts.named = true, nil + return + } + ts.named = slices.Clone(named) +} + +// tiles sends what the tiles class owes: the trailing tile as its buckets +// close, a tile that closed, and at connect or a resync the tiles of the +// last 48 hours the service lacks (docs/protocol-cloud.md, tiles). It reads +// the ledger only when a bucket closed or a pass is due. +func (w *writer) tiles(snap grantSnapshot, st *cadence, now time.Time) error { + l := w.link + hs, ok := l.opts.Site.(HistorySite) + // Tiles are dated by wall time, so a box without NTP sends none. + if !ok || !snap.allowed(cloudwire.MsgTiles) || !l.clockSynced() { + return nil + } + ts := &st.tiles + if now.Before(ts.retryAt) { + return nil + } + res := cloudwire.TileRes + step, span := protocol.ResolutionStepMs[res], protocol.ResolutionTileSpanMs[res] + wall := l.opts.Now().UnixMilli() + closedThrough := (wall - cloudwire.TileSettleMs) / step * step + open := protocol.TileStartFor(res, closedThrough) + oldest := wall - cloudwire.TileKeepMs + + pass := ts.all || len(ts.named) > 0 || !now.Before(ts.recheckAt) + if pass && (ts.all || slices.Contains(ts.named, protocol.TileIDFor(res, 1, open))) { + // The trailing tile goes whole again. + ts.trail, ts.through = open, open + } + if ts.trail != open { + // The trailing tile closed since its last piece: it goes closed. + if prev := ts.trail; prev != 0 && prev < open && prev+span > oldest { + if sent, err := w.closedTile(snap, hs, ts, prev, closedThrough, false); err != nil || !sent { + return err + } + } + ts.trail, ts.through, ts.appends = open, open, 0 + } + if closedThrough > ts.through { + whole := ts.through == open || ts.appends >= cloudwire.TileAppendsMax + from := ts.through + if whole { + from = open + } + c, err := w.pack(hs, ts, open, from, closedThrough) + if err != nil { + return nil + } + first := int((from - open) / step) + piece := cloudwire.Piece(c, first, int((closedThrough-open)/step)) + if whole { + piece = cloudwire.Piece(c, 0, int(protocol.PointsPerTile(res))) + } + if sent, err := w.sendTile(snap, ts, &piece); err != nil || !sent { + return err + } + ts.through = closedThrough + if whole { + ts.appends = 0 + } else { + ts.appends++ + } + } + + if !pass { + return nil + } + // Every closed tile of the window, newest first, that the service does + // not hold at this etag, or every one named. + for start := open - span; start+span > oldest; start -= span { + id := protocol.TileIDFor(res, 1, start) + if !ts.all && len(ts.named) > 0 && !slices.Contains(ts.named, id) { + continue + } + if sent, err := w.closedTile(snap, hs, ts, start, closedThrough, ts.force); err != nil || !sent { + return err + } + } + ts.all, ts.force, ts.named = false, false, nil + ts.recheckAt = now.Add(tilesRecheckEvery) + return nil +} + +// closedTile packs the tile at start whole and sends it unless the service +// holds it at that etag and force is false. sent is false when nothing could +// go and the caller should stop for now; a tile that needs no sending counts +// as sent. +func (w *writer) closedTile(snap grantSnapshot, hs HistorySite, ts *tileState, start, closedThrough int64, force bool) (bool, error) { + span := protocol.ResolutionTileSpanMs[cloudwire.TileRes] + c, err := w.pack(hs, ts, start, start, start+span) + if err != nil { + return false, nil + } + c.Partial = start+span > closedThrough + if c.Partial || (!force && ts.have[c.TileID] == c.Etag) { + return true, nil + } + body := cloudwire.Tiles{TileID: c.TileID, Etag: c.Etag, Res: c.Res, StartMs: c.StartMs, StepMs: c.StepMs, + Series: c.Series, Data: c.Data} + sent, err := w.sendTile(snap, ts, &body) + if sent { + ts.have[c.TileID] = c.Etag + } + return sent, err +} + +// pack reads [from, to) of the tile at start from the ledger and packs the +// tile as the app protocol does, so a closed tile is the same bytes as over +// Noise. Buckets from to on are left out: they have not closed. +func (w *writer) pack(hs HistorySite, ts *tileState, start, from, to int64) (protocol.HistChunk, error) { + l := w.link + res := cloudwire.TileRes + ctx, cancel := context.WithTimeout(context.Background(), tilesReadTimeout) + defer cancel() + windows, err := hs.HistoryWindow(ctx, cloudwire.TileSeries, protocol.ResolutionStepMs[res], from, to) + if err != nil { + l.log.Debug("FTW Cloud tiles: the ledger did not answer", "err", err) + ts.retryAt = time.Now().Add(tilesRetryAfter) + return protocol.HistChunk{}, err + } + for name, samples := range windows { + windows[name] = slices.DeleteFunc(slices.Clone(samples), func(s protocol.HistorySample) bool { + return s.StartMs < from || s.StartMs >= to + }) + } + return protocol.PackTile(res, 1, start, cloudwire.TileSeries, windows, to), nil +} + +// sendTile sends one tile under the byte budget. Past it, tiles wait a +// minute and go again from where they were. +func (w *writer) sendTile(snap grantSnapshot, ts *tileState, body *cloudwire.Tiles) (bool, error) { + sent, err := w.sendBody(snap, cloudwire.MsgTiles, body, false) + if err == nil && !sent { + ts.retryAt = time.Now().Add(tilesRetryAfter) + } + return sent, err +} diff --git a/go/internal/cloudlink/tiles_test.go b/go/internal/cloudlink/tiles_test.go new file mode 100644 index 000000000..748508bd4 --- /dev/null +++ b/go/internal/cloudlink/tiles_test.go @@ -0,0 +1,410 @@ +package cloudlink + +import ( + "bytes" + "context" + "crypto/ecdsa" + "encoding/binary" + "errors" + "slices" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// tileWall is 2026-10-12 17:12:00 UTC: the trailing tile started at 12:00, +// and the bucket that ended at 17:10 has settled. +var tileWall = time.UnixMilli(1_791_806_400_000 + (5*60+12)*60_000) + +// historySite is the fake house with a ledger that has a bucket for every +// five minutes up to the wall clock, the one still filling included, as +// the real one has. +type historySite struct { + *fakeSite + mu sync.Mutex + wall time.Time + // bump is added to every value from bumpFrom on, as a ledger repair + // would change them. + bump float64 + bumpFrom int64 + failing bool + reads int +} + +func (s *historySite) now() time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return s.wall +} + +func (s *historySite) set(wall time.Time) { + s.mu.Lock() + s.wall = wall + s.mu.Unlock() +} + +// value is the ledger's mean for one series and bucket. +func (s *historySite) value(name string, start int64) float64 { + i := float64(start/300_000%10_000) + float64(len(name)) + if start >= s.bumpFrom { + i += s.bump + } + switch name { + case "pv_w": + return -i + case "battery_w": + return 2 * i + } + return i +} + +func (s *historySite) HistoryWindow(_ context.Context, names []string, stepMs, fromMs, toMs int64) (map[string][]protocol.HistorySample, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.reads++ + if s.failing { + return nil, errors.New("the ledger is busy") + } + out := map[string][]protocol.HistorySample{} + for at := fromMs / stepMs * stepMs; at < toMs && at <= s.wall.UnixMilli(); at += stepMs { + for _, name := range names { + out[name] = append(out[name], protocol.HistorySample{StartMs: at, W: s.value(name, at)}) + } + } + return out, nil +} + +// noise is the tile the box serves over Noise at the same moment. +func (s *historySite) noise(t *testing.T, start int64) protocol.HistChunk { + t.Helper() + span := protocol.ResolutionTileSpanMs[cloudwire.TileRes] + w, err := s.HistoryWindow(context.Background(), cloudwire.TileSeries, 300_000, start, start+span) + if err != nil { + t.Fatal(err) + } + return protocol.PackTile(cloudwire.TileRes, 1, start, cloudwire.TileSeries, w, s.now().UnixMilli()) +} + +func newTileHarness(t *testing.T, svc *fakeService, tweak func(*Options)) (*harness, *historySite) { + t.Helper() + hs := &historySite{wall: tileWall, bumpFrom: 1 << 62} + h := newHarnessWith(t, svc, []*ecdsa.PublicKey{&svc.key.PublicKey}, nil, func(o *Options) { + hs.fakeSite = o.Site.(*fakeSite) + o.Site, o.Now = hs, hs.now + if tweak != nil { + tweak(o) + } + }) + return h, hs +} + +// tilesSent decodes every tiles body the service has received, in order. +func tilesSent(t *testing.T, svc *fakeService) []cloudwire.Tiles { + t.Helper() + var out []cloudwire.Tiles + for _, env := range svc.received() { + if env.T != cloudwire.MsgTiles { + continue + } + var b cloudwire.Tiles + if err := cloudwire.Unmarshal(env.B, &b); err != nil { + t.Fatal(err) + } + if err := b.Check(); err != nil { + t.Fatalf("the box sent a tile of no shape: %v", err) + } + out = append(out, b) + } + return out +} + +// waitTiles waits until the service has n tiles bodies, and a little more, +// so one too many shows too. +func waitTiles(t *testing.T, svc *fakeService, n int) []cloudwire.Tiles { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for len(tilesSent(t, svc)) < n && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + time.Sleep(150 * time.Millisecond) + got := tilesSent(t, svc) + if len(got) != n { + t.Fatalf("the service has %d tiles, want %d: %v", len(got), n, tileNames(got)) + } + return got +} + +func tileNames(ts []cloudwire.Tiles) []string { + var out []string + for _, b := range ts { + out = append(out, b.TileID+"@"+time.UnixMilli(b.StartMs).UTC().Format("02 15:04")) + } + return out +} + +func point(b cloudwire.Tiles, series string, i int) int32 { + s := slices.Index(b.Series, series) + return int32(binary.LittleEndian.Uint32(b.Data[(s*b.Points()+i)*4:])) +} + +func tileID(start int64) string { return protocol.TileIDFor(cloudwire.TileRes, 1, start) } + +const ( + trailStart = int64(1_791_806_400_000) // 2026-10-12 12:00 UTC + tileSpan = int64(43_200_000) +) + +func grantHistory(t *testing.T, h *harness) { + t.Helper() + h.grant(t, protocol.ScopeLiveRead, protocol.ScopeHistoryRead) +} + +func TestAtConnectTheTrailingTileGoesWholeAndTheClosedOnesAsOverNoise(t *testing.T) { + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, nil) + grantHistory(t, h) + got := waitTiles(t, svc, 5) + + whole := got[0] + if whole.TileID != tileID(trailStart) || !whole.Partial || !whole.Whole() || whole.Points() != 144 { + t.Fatalf("first tile %v, want the trailing tile whole", tileNames(got[:1])) + } + // 17:05 has closed; 17:10 has not settled, and the bucket still filling + // in the ledger stays out. + at1705, at1710 := int((17*60+5-12*60)/5), int((17*60+10-12*60)/5) + if v := point(whole, "grid_w", at1705); v != protocol.ClampSample(hs.value("grid_w", trailStart+int64(at1705)*300_000)) { + t.Fatalf("17:05 reads %d", v) + } + if point(whole, "grid_w", at1710) != protocol.MissingSample || point(whole, "load_w", 143) != protocol.MissingSample { + t.Fatal("a bucket that has not closed went as a value") + } + + // Then the closed tiles of the last 48 hours, newest first, each the + // Noise tile byte for byte. + want := []int64{trailStart - tileSpan, trailStart - 2*tileSpan, trailStart - 3*tileSpan, trailStart - 4*tileSpan} + for i, start := range want { + b := got[i+1] + n := hs.noise(t, start) + if b.Partial || b.TileID != n.TileID || b.Etag != n.Etag || !bytes.Equal(b.Data, n.Data) || b.StartMs != n.StartMs { + t.Fatalf("closed tile %d is %v, not the Noise tile %s", i, tileNames([]cloudwire.Tiles{b}), n.TileID) + } + } + // Caps say so. + var hello cloudwire.Hello + payload, _ := cloudwire.DecodeFrame(svc.hello) + env, _ := cloudwire.DecodeEnvelope(payload) + if cloudwire.Unmarshal(env.B, &hello) != nil || !slices.Contains(hello.Caps, cloudwire.MsgTiles) { + t.Fatalf("hello.caps %v lacks tiles", hello.Caps) + } +} + +func TestEachClosedBucketGoesAsAnAppendUntilTheTileGoesWholeAgain(t *testing.T) { + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, nil) + grantHistory(t, h) + waitTiles(t, svc, 5) + + // The bucket 17:10–17:15 settles at 17:15:30. + hs.set(tileWall.Add(3*time.Minute + 29*time.Second)) + time.Sleep(200 * time.Millisecond) + if n := len(tilesSent(t, svc)); n != 5 { + t.Fatalf("a tile went before its bucket settled: %d", n) + } + for i := 1; i <= cloudwire.TileAppendsMax+1; i++ { + hs.set(tileWall.Add(time.Duration(i)*5*time.Minute - 2*time.Minute + 31*time.Second)) + got := waitTiles(t, svc, 5+i) + b := got[len(got)-1] + if i <= cloudwire.TileAppendsMax { + start := trailStart + int64((17*60+10-12*60+5*(i-1)))*60_000 + if !b.Partial || b.Whole() || b.Points() != 1 || b.StartMs != start { + t.Fatalf("append %d is %v", i, tileNames([]cloudwire.Tiles{b})) + } + if point(b, "pv_w", 0) != protocol.ClampSample(hs.value("pv_w", start)) { + t.Fatalf("append %d carries %d", i, point(b, "pv_w", 0)) + } + continue + } + if !b.Partial || !b.Whole() || b.Points() != 144 { + t.Fatalf("after %d appends the tile went as %v, not whole", cloudwire.TileAppendsMax, tileNames([]cloudwire.Tiles{b})) + } + } + // The pieces lay into the tile the box holds. + laid := cloudwire.Lay(tilesSent(t, svc)) + if last := laid[len(laid)-1]; last.TileID != tileID(trailStart) || !last.Partial { + t.Fatalf("laid %v", last.TileID) + } +} + +func TestATileGoesClosedWhenItsLastBucketCloses(t *testing.T) { + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, nil) + hs.set(time.UnixMilli(trailStart + tileSpan - 2*60_000)) // 23:58 + grantHistory(t, h) + waitTiles(t, svc, 5) + + hs.set(time.UnixMilli(trailStart + tileSpan + 31_000)) // 00:00:31 + got := waitTiles(t, svc, 6) + closed := got[5] + n := hs.noise(t, trailStart) + if closed.Partial || closed.TileID != tileID(trailStart) || closed.Etag != n.Etag { + t.Fatalf("at midnight %v went, not the closed tile", tileNames(got[5:])) + } + // Nothing of the new tile until its first bucket settles. + hs.set(time.UnixMilli(trailStart + tileSpan + 5*60_000 + 31_000)) + got = waitTiles(t, svc, 7) + first := got[6] + if first.TileID != tileID(trailStart+tileSpan) || !first.Whole() || !first.Partial || + point(first, "grid_w", 0) == protocol.MissingSample || point(first, "grid_w", 1) != protocol.MissingSample { + t.Fatalf("the new tile began as %v", tileNames(got[6:])) + } +} + +func TestTheServicesHaveSavesItsClosedTiles(t *testing.T) { + svc := newFakeService(t) + probe := &historySite{wall: tileWall, bumpFrom: 1 << 62} + held := probe.noise(t, trailStart-tileSpan) + svc.have = map[string]string{ + held.TileID: held.Etag, + tileID(trailStart - 3*tileSpan): "00000000", // an older etag + "5m/1/1": "ffffffff", // nothing the box has + } + h, _ := newTileHarness(t, svc, nil) + grantHistory(t, h) + got := waitTiles(t, svc, 4) + ids := []string{got[1].TileID, got[2].TileID, got[3].TileID} + if slices.Contains(ids, held.TileID) || !slices.Contains(ids, tileID(trailStart-3*tileSpan)) { + t.Fatalf("sent %v with have %v", ids, svc.have) + } +} + +func TestAResyncOfTilesSendsThemAgainOnceInFiveMinutes(t *testing.T) { + svc := newFakeService(t) + conns := make(chan uplinkConn, 4) + svc.afterHello = func(conn uplinkConn, _ string) { conns <- conn } + h, _ := newTileHarness(t, svc, nil) + grantHistory(t, h) + waitTiles(t, svc, 5) + conn := <-conns + + resync := func(named ...string) { + svc.write(conn, svc.key, svc.home, cloudwire.MsgResync, cloudwire.Resync{What: []string{cloudwire.ResyncTiles}, Tiles: named}) + } + resync() + got := waitTiles(t, svc, 10) + if !got[5].Whole() || !got[5].Partial || got[6].Partial || got[9].TileID != tileID(trailStart-4*tileSpan) { + t.Fatalf("a resync sent %v", tileNames(got[5:])) + } + // A second within five minutes is ignored. + resync() + waitTiles(t, svc, 10) + + // One that names a tile sends that one alone. + h.link.mu.Lock() + h.link.resyncAt = time.Time{} + h.link.mu.Unlock() + resync(tileID(trailStart - 2*tileSpan)) + got = waitTiles(t, svc, 11) + if got[10].TileID != tileID(trailStart-2*tileSpan) || got[10].Partial { + t.Fatalf("a named resync sent %v", tileNames(got[10:])) + } +} + +func TestAReconnectSendsWhatTheServiceLacks(t *testing.T) { + svc := newFakeService(t) + svc.kick = 1 + h, _ := newTileHarness(t, svc, nil) + grantHistory(t, h) + // The first connection drops right after hello_ok, before any tile; the + // second sends the trailing tile and the closed ones. + got := waitTiles(t, svc, 5) + if svc.conns < 2 || !got[0].Whole() || !got[0].Partial { + t.Fatalf("after a reconnect: %d connections, %v", svc.conns, tileNames(got)) + } +} + +func TestTheHourlyPassResendsATileTheLedgerRepaired(t *testing.T) { + old := tilesRecheckEvery + tilesRecheckEvery = 300 * time.Millisecond + t.Cleanup(func() { tilesRecheckEvery = old }) + + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, nil) + grantHistory(t, h) + waitTiles(t, svc, 5) + + // The ledger fills a gap two tiles back. + repaired := trailStart - 2*tileSpan + hs.mu.Lock() + hs.bump, hs.bumpFrom = 7, repaired+tileSpan/2 + hs.mu.Unlock() + // The trailing tile changes too, but it goes only as it closes. + got := waitTiles(t, svc, 7) + again := got[5:] + ids := []string{again[0].TileID, again[1].TileID} + if !slices.Contains(ids, tileID(repaired)) || !slices.Contains(ids, tileID(repaired+tileSpan)) { + t.Fatalf("after the repair %v went", tileNames(again)) + } + time.Sleep(500 * time.Millisecond) + if n := len(tilesSent(t, svc)); n != 7 { + t.Fatalf("an unchanged tile went again: %d", n) + } +} + +func TestNoTilesWithoutTheHistoryScopeAClockOrTheBudget(t *testing.T) { + cases := map[string]struct { + scopes []string + tweak func(*Options) + budget bool + }{ + "live only": {scopes: []string{protocol.ScopeLiveRead}}, + "no clock": {scopes: []string{protocol.ScopeLiveRead, protocol.ScopeHistoryRead}, + tweak: func(o *Options) { o.ClockSynced = func() bool { return false } }}, + "past the budget": {scopes: []string{protocol.ScopeLiveRead, protocol.ScopeHistoryRead}, budget: true}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, c.tweak) + if c.budget { + h.link.mu.Lock() + h.link.budget.add(h.link.uptimeMs(), budgetBytes) + h.link.mu.Unlock() + } + h.grant(t, c.scopes...) + svc.waitFor(cloudwire.MsgRoster, 5*time.Second) + time.Sleep(time.Second) + if n := len(tilesSent(t, svc)); n != 0 { + t.Fatalf("%d tiles went", n) + } + if name == "live only" { + hs.mu.Lock() + reads := hs.reads + hs.mu.Unlock() + if reads != 0 { + t.Fatalf("the ledger was read %d times for nothing", reads) + } + } + }) + } +} + +func TestALedgerThatFailsHoldsTilesBackAndNothingElse(t *testing.T) { + svc := newFakeService(t) + h, hs := newTileHarness(t, svc, nil) + hs.failing = true + grantHistory(t, h) + svc.waitFor(cloudwire.MsgLive, 5*time.Second) + svc.waitFor(cloudwire.MsgHB, 20*time.Second) + if n := len(tilesSent(t, svc)); n != 0 { + t.Fatalf("%d tiles went from a failing ledger", n) + } + hs.mu.Lock() + reads := hs.reads + hs.mu.Unlock() + if reads > 2 { + t.Fatalf("a failing ledger was read %d times within its retry", reads) + } +} From 50bb11690400b4c60a8e833a533014842b4edf3e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 23 Oct 2026 16:26:08 +0200 Subject: [PATCH 3/5] docs(cloud): pieces lay in the order the box sent them, by wallMs then seq seq starts again when the box does, and a phone may still hold pieces from before the restart. Co-Authored-By: Claude Opus 5.5 --- docs/protocol-cloud.md | 7 ++++--- go/protocol/cloudwire/tiles.go | 13 +++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/protocol-cloud.md b/docs/protocol-cloud.md index e4c724401..88d5c870d 100644 --- a/docs/protocol-cloud.md +++ b/docs/protocol-cloud.md @@ -668,8 +668,9 @@ count against the byte budget: about 100 KB a day. `cloudwire.Lay` puts what a receiver holds back together: a closed tile as it is, and the trailing tile from its newest whole piece with each later -append laid over it, in `seq` order. The trailing tile stays partial, and -its etag is never offered as held. +append laid over it, in the order the box sent them, by `wallMs` and then +`seq`, since `seq` starts again when the box does. The trailing tile stays +partial, and its etag is never offered as held. ### `hb`: every device in the roster @@ -1002,7 +1003,7 @@ the table. |---|---|---| | `watch` | Any device | `{hz}`: 1 when the phone looks at the home, 0 when it looks away. It holds until the next `watch` or the end of the session | | `tun` | A device with `tun` | `{sid, d}`. The phone's session with its box. See [Tunnel](#tunnel). The service answers `tun` the same way | -| `tiles.get` | Devices with `ftw.history.read` | `{have}`, tile id to etag of the closed tiles the phone holds. The service answers with the `tiles` envelopes the phone lacks: each closed tile it holds that `have` does not name at that etag, and every piece of the trailing tile, oldest tile first and each tile's pieces in `seq` order. Then `tiles.end {n}`, where `n` counts them. It answers this while the box is away too, and a device without the scope gets `tiles.end {n: 0}` | +| `tiles.get` | Devices with `ftw.history.read` | `{have}`, tile id to etag of the closed tiles the phone holds. The service answers with the `tiles` envelopes the phone lacks: each closed tile it holds that `have` does not name at that etag, and every piece of the trailing tile, oldest tile first and each tile's pieces in the order the box sent them. Then `tiles.end {n}`, where `n` counts them. It answers this while the box is away too, and a device without the scope gets `tiles.end {n: 0}` | | `token.get` | Any device | Answered with `token {token, expiresInS}`. A new token replaces the device's old one | | `push.register` | Any device | `{apnsToken, environment, kind, activityId, app}`. One registration per APNs token; a Live Activity and its push-to-start each register their own. `app` is `iphone`, the default when absent, or `watch`, and picks the APNs topic | | `push.unregister` | Any device | `{apnsToken}` | diff --git a/go/protocol/cloudwire/tiles.go b/go/protocol/cloudwire/tiles.go index 869072ad6..3fbac9e82 100644 --- a/go/protocol/cloudwire/tiles.go +++ b/go/protocol/cloudwire/tiles.go @@ -174,12 +174,13 @@ func Piece(c protocol.HistChunk, from, to int) Tiles { Series: slices.Clone(c.Series), Data: data, Partial: true} } -// Lay puts tiles back together, in the order the box sent them: a closed tile -// as it is; the trailing tile from its newest whole piece, with each later -// append laid over it. A piece of a tile already closed changes nothing, and -// neither does a body that fails Check. The result has one chunk per tile, -// oldest first, each with every point; a trailing tile stays partial, its -// etag that of the data as laid, never one to offer as held. +// Lay puts tiles back together, in the order the box sent them, by wallMs +// and then seq, which starts again with the box: a closed tile as it is; the +// trailing tile from its newest whole piece, with each later append laid +// over it. A piece of a tile already closed changes nothing, and neither does +// a body that fails Check. The result has one chunk per tile, oldest first, +// each with every point; a trailing tile stays partial, its etag that of the +// data as laid, never one to offer as held. func Lay(bodies []Tiles) []protocol.HistChunk { byID := map[string]*protocol.HistChunk{} for _, t := range bodies { From 1147d809abbe3300291451f8b4a4113e9dd75b5f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 23 Oct 2026 16:36:11 +0200 Subject: [PATCH 4/5] fix(cloudwire): a later closed tile replaces an earlier one in Lay The box sends a closed tile again when its ledger changed under it, as in the hourly pass. Lay kept the first, so a receiver drew the old gap. Only a piece of a closed tile is ignored now. Co-Authored-By: Claude Opus 5.5 --- go/protocol/cloudwire/tiles.go | 9 ++++++--- go/protocol/cloudwire/tiles_test.go | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go/protocol/cloudwire/tiles.go b/go/protocol/cloudwire/tiles.go index 3fbac9e82..346c07905 100644 --- a/go/protocol/cloudwire/tiles.go +++ b/go/protocol/cloudwire/tiles.go @@ -177,8 +177,8 @@ func Piece(c protocol.HistChunk, from, to int) Tiles { // Lay puts tiles back together, in the order the box sent them, by wallMs // and then seq, which starts again with the box: a closed tile as it is; the // trailing tile from its newest whole piece, with each later append laid -// over it. A piece of a tile already closed changes nothing, and neither does -// a body that fails Check. The result has one chunk per tile, oldest first, +// over it; a later closed tile replaces an earlier one. A piece of a tile +// already closed changes nothing, and neither does a body that fails Check. The result has one chunk per tile, oldest first, // each with every point; a trailing tile stays partial, its etag that of the // data as laid, never one to offer as held. func Lay(bodies []Tiles) []protocol.HistChunk { @@ -188,10 +188,13 @@ func Lay(bodies []Tiles) []protocol.HistChunk { continue } held := byID[t.TileID] - if held != nil && !held.Partial { + if held != nil && !held.Partial && t.Partial { + // A piece of a tile already closed. continue } if !t.Partial { + // A closed tile, or a later one for the same tile: the box sends + // a closed tile again when the ledger changed under it. c := t.Chunk() byID[t.TileID] = &c continue diff --git a/go/protocol/cloudwire/tiles_test.go b/go/protocol/cloudwire/tiles_test.go index 18fb4eae2..57e22565b 100644 --- a/go/protocol/cloudwire/tiles_test.go +++ b/go/protocol/cloudwire/tiles_test.go @@ -193,6 +193,15 @@ func TestLayKeepsTheNewestWholeAndTheClosedTile(t *testing.T) { if laid[0].Partial || laid[0].Etag != v[0].body.Etag { t.Fatal("a piece reopened a closed tile") } + // A closed tile sent again after the ledger changed replaces the first. + fixed := v[0].body + fixed.Data = slices.Clone(fixed.Data) + fixed.Data[0] ^= 1 + fixed.Etag = protocol.EtagOf(fixed.Data) + laid = Lay([]Tiles{v[0].body, fixed, piece}) + if laid[0].Partial || laid[0].Etag != fixed.Etag { + t.Fatal("a later closed tile did not replace the first") + } // A body that fails Check is skipped. bad := v[3].body bad.Etag = "00000000" From b10a69394f99cc57d11d16184bfbe6e58ae10884 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 23 Oct 2026 17:36:09 +0200 Subject: [PATCH 5/5] fix(cloud): tiles after review: exact series, a 4 KiB bound, history back in the grant - Check takes the four series and no other, from the registry's names, and TileEnvelopeMax bounds an envelope, so a box cannot pad its tiles. - When history leaves the grant the link forgets what it sent; when it comes back every tile goes again on the same link. - The ledger is read at most once a tick, one closed tile at a time, and the grant is checked again before each tile goes. - cloudlink.PackTile is the one packing path; a test holds it to the Noise tile through appproto's buildTile. - The doc says what a phone really gets: tiles until the roster runs out while the box is away, and when FTW Cloud sends resync for tiles. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0008-ftw-cloud.md | 2 +- docs/protocol-cloud.md | 48 ++++++--- go/internal/appproto/cloud_tile_test.go | 43 ++++++++ go/internal/cloudlink/tiles.go | 132 +++++++++++++++--------- go/internal/cloudlink/tiles_test.go | 24 +++++ go/protocol/cloudwire/tiles.go | 29 +++--- go/protocol/cloudwire/tiles_test.go | 25 +++++ 7 files changed, 222 insertions(+), 81 deletions(-) create mode 100644 go/internal/appproto/cloud_tile_test.go diff --git a/docs/adr/0008-ftw-cloud.md b/docs/adr/0008-ftw-cloud.md index a5c14502d..6915dbf55 100644 --- a/docs/adr/0008-ftw-cloud.md +++ b/docs/adr/0008-ftw-cloud.md @@ -212,7 +212,7 @@ The privacy notice says so. | Store | Kept | |---|---| | Live state | Memory only; readings dropped an hour after the box goes quiet | -| History tiles | Memory only; each 12-hour tile of 5-minute history until 48 hours after it ends, box online or not | +| History tiles | Memory only; each 12-hour tile of 5-minute history until 48 hours after it ends or the roster runs out, box online or not | | Registry | Until purge, or 90 days without a roster | | APNs registrations | Until purge, or 30 days without the box | | Events | A one-hour ring in memory | diff --git a/docs/protocol-cloud.md b/docs/protocol-cloud.md index 88d5c870d..a2b1834bc 100644 --- a/docs/protocol-cloud.md +++ b/docs/protocol-cloud.md @@ -620,7 +620,7 @@ when its own rules (enabled, cooldown) allow. Delivery is best effort. ### `tiles`: `ftw.history.read` -The last 48 hours of the box's 5-minute history, so a phone can draw it while +The box's 5-minute history of the last 48 hours, so a phone can draw it while the box cannot answer. The body is the app protocol's `hist.chunk` body: | Field | Type | Meaning | @@ -630,12 +630,13 @@ the box cannot answer. The body is the app protocol's `hist.chunk` body: | `res` | text | `5m` | | `startMs` | int | Wall time of the first point. The tile's start, except in an append | | `stepMs` | uint | 300,000 | -| `series` | [text] | `grid_w`, `pv_w`, `battery_w` and `load_w`, in that order: the series the app asks for over Noise | +| `series` | [text] | `grid_w`, `pv_w`, `battery_w` and `load_w`, in that order and no other: the series the app asks for over Noise | | `data` | bstr | int32 LE, one block per series, the app protocol's missing value where the box has no record | | `partial` | bool | The tile is still filling | A body is one of three shapes. `cloudwire.Tiles.Check` holds them, and -`cloud-tile-vectors.json` has one of each. +`cloud-tile-vectors.json` has one of each. A tiles envelope is at most 4 KiB +(`TileEnvelopeMax`); a whole tile is about 2.6 KB. - **Closed:** `partial` false, `startMs` the tile's start, all 144 points. It has the same tile id, etag and bytes as the box serves over Noise for a @@ -661,15 +662,28 @@ has committed it. Then: and every closed tile of the last 48 hours that `hello_ok.have` lacks at that etag, newest first. `resync {what: [tiles]}` sends the tiles it names, or all of them when it names none, whatever `have` says. - -The box sends tiles only while NTP has synced its clock, since tiles are -dated by wall time, and only under a grant with `ftw.history.read`. They -count against the byte budget: about 100 KB a day. +- When `ftw.history.read` leaves the grant it forgets what it sent, since the + service drops what it held; when it comes back, the box sends every tile + again, as at connect. + +The box reads its ledger at most once per tick, one tile at a time, so a +reading never waits long behind a pass. It sends tiles only while NTP has +synced its clock, since tiles are dated by wall time, and only under a grant +with `ftw.history.read`, which it checks again before each one. They count +against the byte budget: about 100 KB a day. + +The service keeps each tile until 48 hours after it ends, so up to 60 hours +of history, and drops them all when the roster runs out. A phone reads them +only while the roster is in force, so with the box away that is 18 to 24 +hours after the box was last heard. The service sends the box +`resync {what: [tiles]}`, at most once per 5 minutes, when a roster gives +history back and when room frees after it dropped a tile for lack of it. `cloudwire.Lay` puts what a receiver holds back together: a closed tile as it is, and the trailing tile from its newest whole piece with each later append laid over it, in the order the box sent them, by `wallMs` and then -`seq`, since `seq` starts again when the box does. The trailing tile stays +`seq`, since `seq` starts again when the box does. A later closed tile +replaces an earlier one, never an older one a later. The trailing tile stays partial, and its etag is never offered as held. ### `hb`: every device in the roster @@ -894,7 +908,7 @@ a lane 0 frame closes the uplink. A fuzz test covers it. | `challenge` | `{v, nonce}` | First message. Unsigned | | `hello_ok` | See Handshake | Signed by the service key | | `watch` | `{hz: 1}` | Raises the period to 1 s for 60 s. While any phone of the home looks, the service sends it at once, every 50 s after that, and again when the box connects. Without it the box returns to 5 s. The box ignores any other `hz` | -| `resync` | `{what, tiles}` | `what` holds `full`, `tiles` or `roster`; `tiles` lists tile ids, and none means every tile of the last 48 hours. The box honours at most one per 5 minutes and ignores the rest | +| `resync` | `{what, tiles}` | `what` holds `full`, `tiles` or `roster`; `tiles` lists tile ids, and none means every tile of the last 48 hours. The box honours at most one per 5 minutes and ignores the rest. The service sends one for tiles when a roster gives history back, and when room frees after it dropped a tile | | `purged` | `{atMs, what, grantId, statsId}` | Signed by the service key. `what` is `home`, `copy` or `stats`. Sent once every deletion it names is done | | `tun` | `{dev, sid, d}` | A phone's own session with the box. See [Tunnel](#tunnel). Only to a box that lists `tun` in `hello.caps` | @@ -1540,7 +1554,8 @@ writes do not gain access. CGNAT is not a private network for this choice. the box is offline, this happens on reconnect. - **Silence:** after an hour without the box's socket, the service serves only `lastSeenMs`, no readings, and the tiles, each until 48 hours after its - end. A tile is dated history and never reads as now. When the roster runs out, it closes sessions + end or until the roster runs out. A tile is dated history and never reads + as now. When the roster runs out, it closes sessions and refuses reads. After 30 days it drops tokens and APNs registrations. After 90 days without a valid roster it deletes the home as in a purge. - **A new home** that sends no roster within 24 hours is deleted. @@ -1622,10 +1637,11 @@ applied. Per home, the service keeps in memory the newest envelope of each class, the newest `live` that carried `sources`, 48 hours of tiles and a one-hour ring of events. For tiles that is at most five closed tiles and 16 pieces of the -trailing tile, each as the box signed it; a tile that fails -`cloudwire.Tiles.Check` is dropped and counted. All homes together hold at -most a fixed number of tile bytes, past which new tiles are dropped and -counted. `have` in `hello_ok` names the closed tiles held. It writes no live readings to disk; the statistics files hold the +trailing tile, each as the box signed it, and at most 64 KiB per home; a +tile that fails `cloudwire.Tiles.Check` or is over 4 KiB is dropped and +counted. All homes together hold at most a fixed number of tile bytes, past +which new tiles are dropped and counted. `have` in `hello_ok` names the +closed tiles held. It writes no live readings to disk; the statistics files hold the only readings it stores. Its registry holds the home id, the box key, the roster, suspensions, the current and deleted `statsId`s, APNs registrations and token hashes. @@ -1718,8 +1734,8 @@ constants, so `srcfl/ftw-cloud` need not spell them by hand. | Extension token | 7 days | | Alert age for buttons | 10 minutes | | `box.unreachable` after | 600 s | -| Readings served after silence | 1 hour; tiles until 48 hours after their end | -| Tiles | `5m`, 12-hour tiles, the last 48 hours: at most 5 closed and 16 pieces of the trailing one per home | +| Readings served after silence | 1 hour; tiles until 48 hours after their end, or until the roster runs out | +| Tiles | `5m`, 12-hour tiles, the last 48 hours: at most 5 closed and 16 pieces of the trailing one, 64 KiB, per home; 4 KiB an envelope | | Bucket counted closed | 30 s after it ends | | Trailing tile sent whole | At each connect, after 11 appends, and when its first bucket closes | | Closed tiles packed again | Every hour, sent when the etag changed | diff --git a/go/internal/appproto/cloud_tile_test.go b/go/internal/appproto/cloud_tile_test.go new file mode 100644 index 000000000..a86127d70 --- /dev/null +++ b/go/internal/appproto/cloud_tile_test.go @@ -0,0 +1,43 @@ +package appproto + +import ( + "bytes" + "context" + "testing" + + "github.com/srcfl/ftw/go/internal/cloudlink" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// ledgerSite is a history provider as the cloud link reads it. +type ledgerSite struct{ HistoryProvider } + +func (s ledgerSite) HistoryWindow(ctx context.Context, names []string, stepMs, fromMs, toMs int64) (map[string][]HistorySample, error) { + return s.Window(ctx, names, stepMs, fromMs, toMs) +} + +// A closed tile FTW Cloud gets is the tile the box serves over Noise for the +// same series: the same id, etag and bytes, holes included, read through the +// two real paths. +func TestACloudTileIsTheNoiseTile(t *testing.T) { + start := tileStartFor(ResN5m, 1_760_000_000_000-24*3_600_000) + span := ResolutionTileSpanMs[ResN5m] + ledger := &fakeLedger{ + values: map[string]float64{"grid_w": 1200.4, "pv_w": -800, "battery_w": 300, "load_w": 1700.6}, + holes: map[int64]bool{start + 5*300_000: true, start + 90*300_000: true}, + } + h := &Handler{cfg: Config{History: ledger}} + plan := planHistoryQuery(ResN5m, start, start+span, defaultMaxPoints) + noise, err := h.buildTile(context.Background(), cloudwire.TileSeries, plan, plan.tiles[0], start+2*span) + if err != nil { + t.Fatal(err) + } + cloud, err := cloudlink.PackTile(context.Background(), ledgerSite{ledger}, start, start, start+span) + if err != nil { + t.Fatal(err) + } + if noise.TileID != cloud.TileID || noise.Etag != cloud.Etag || !bytes.Equal(noise.Data, cloud.Data) || + noise.Partial || cloud.Partial || noise.StepMs != cloud.StepMs { + t.Fatalf("Noise %s %s, FTW Cloud %s %s", noise.TileID, noise.Etag, cloud.TileID, cloud.Etag) + } +} diff --git a/go/internal/cloudlink/tiles.go b/go/internal/cloudlink/tiles.go index d7ef8b69b..c47d13d86 100644 --- a/go/internal/cloudlink/tiles.go +++ b/go/internal/cloudlink/tiles.go @@ -47,8 +47,12 @@ type tileState struct { // tiles a resync named. all, force bool named []string - recheckAt time.Time - retryAt time.Time + // passing says a pass over the closed tiles of the window is under way, + // and passAt is the start of the next one it looks at, newest first. + passing bool + passAt int64 + recheckAt time.Time + retryAt time.Time } func newTileState(have map[string]string) tileState { @@ -60,9 +64,9 @@ func newTileState(have map[string]string) tileState { } // resync takes the service's resync of tiles: the tiles it names, or every -// one of the window when it names none. +// one of the window when it names none. A pass under way starts again. func (ts *tileState) resync(named []string) { - ts.force = true + ts.force, ts.passing = true, false if len(named) == 0 || len(named) > tilesNamedMax { ts.all, ts.named = true, nil return @@ -73,12 +77,19 @@ func (ts *tileState) resync(named []string) { // tiles sends what the tiles class owes: the trailing tile as its buckets // close, a tile that closed, and at connect or a resync the tiles of the // last 48 hours the service lacks (docs/protocol-cloud.md, tiles). It reads -// the ledger only when a bucket closed or a pass is due. +// the ledger at most once a tick, and only when a bucket closed or a pass is +// under way, so a reading never waits long behind it. func (w *writer) tiles(snap grantSnapshot, st *cadence, now time.Time) error { l := w.link hs, ok := l.opts.Site.(HistorySite) // Tiles are dated by wall time, so a box without NTP sends none. - if !ok || !snap.allowed(cloudwire.MsgTiles) || !l.clockSynced() { + if !ok || !l.clockSynced() { + return nil + } + if !snap.allowed(cloudwire.MsgTiles) { + // Without history in the grant the service drops what it held, so + // when history comes back every tile goes again, as at connect. + st.tiles = newTileState(nil) return nil } ts := &st.tiles @@ -92,17 +103,22 @@ func (w *writer) tiles(snap grantSnapshot, st *cadence, now time.Time) error { open := protocol.TileStartFor(res, closedThrough) oldest := wall - cloudwire.TileKeepMs - pass := ts.all || len(ts.named) > 0 || !now.Before(ts.recheckAt) - if pass && (ts.all || slices.Contains(ts.named, protocol.TileIDFor(res, 1, open))) { - // The trailing tile goes whole again. - ts.trail, ts.through = open, open + // A pass over the closed tiles: at connect, after a resync, and once an + // hour. The trailing tile goes whole again with every tile. + if !ts.passing && (ts.all || len(ts.named) > 0 || !now.Before(ts.recheckAt)) { + ts.passing, ts.passAt = true, open-span + if ts.all || slices.Contains(ts.named, protocol.TileIDFor(res, 1, open)) { + ts.trail, ts.through = open, open + } } if ts.trail != open { // The trailing tile closed since its last piece: it goes closed. if prev := ts.trail; prev != 0 && prev < open && prev+span > oldest { - if sent, err := w.closedTile(snap, hs, ts, prev, closedThrough, false); err != nil || !sent { - return err + sent, err := w.closedTile(hs, ts, prev, closedThrough, false) + if sent { + ts.trail, ts.through, ts.appends = open, open, 0 } + return err } ts.trail, ts.through, ts.appends = open, open, 0 } @@ -116,76 +132,74 @@ func (w *writer) tiles(snap grantSnapshot, st *cadence, now time.Time) error { if err != nil { return nil } - first := int((from - open) / step) - piece := cloudwire.Piece(c, first, int((closedThrough-open)/step)) + piece := cloudwire.Piece(c, int((from-open)/step), int((closedThrough-open)/step)) if whole { piece = cloudwire.Piece(c, 0, int(protocol.PointsPerTile(res))) } - if sent, err := w.sendTile(snap, ts, &piece); err != nil || !sent { - return err - } - ts.through = closedThrough - if whole { - ts.appends = 0 - } else { - ts.appends++ + sent, err := w.sendTile(ts, &piece) + if sent { + ts.through = closedThrough + if whole { + ts.appends = 0 + } else { + ts.appends++ + } } + return err } - - if !pass { + if !ts.passing { return nil } - // Every closed tile of the window, newest first, that the service does - // not hold at this etag, or every one named. - for start := open - span; start+span > oldest; start -= span { - id := protocol.TileIDFor(res, 1, start) - if !ts.all && len(ts.named) > 0 && !slices.Contains(ts.named, id) { + // One closed tile of the window a tick, newest first, unless the service + // holds it at this etag; or each one a resync named. + for ; ts.passAt+span > oldest; ts.passAt -= span { + if !ts.all && len(ts.named) > 0 && !slices.Contains(ts.named, protocol.TileIDFor(res, 1, ts.passAt)) { continue } - if sent, err := w.closedTile(snap, hs, ts, start, closedThrough, ts.force); err != nil || !sent { - return err + sent, err := w.closedTile(hs, ts, ts.passAt, closedThrough, ts.force) + if sent { + ts.passAt -= span } + return err } - ts.all, ts.force, ts.named = false, false, nil + ts.passing, ts.all, ts.force, ts.named = false, false, false, nil ts.recheckAt = now.Add(tilesRecheckEvery) return nil } // closedTile packs the tile at start whole and sends it unless the service // holds it at that etag and force is false. sent is false when nothing could -// go and the caller should stop for now; a tile that needs no sending counts -// as sent. -func (w *writer) closedTile(snap grantSnapshot, hs HistorySite, ts *tileState, start, closedThrough int64, force bool) (bool, error) { +// go and the caller should try again later; a tile that needs no sending +// counts as sent. +func (w *writer) closedTile(hs HistorySite, ts *tileState, start, closedThrough int64, force bool) (bool, error) { span := protocol.ResolutionTileSpanMs[cloudwire.TileRes] + if start+span > closedThrough { + return true, nil + } c, err := w.pack(hs, ts, start, start, start+span) if err != nil { return false, nil } - c.Partial = start+span > closedThrough - if c.Partial || (!force && ts.have[c.TileID] == c.Etag) { + if !force && ts.have[c.TileID] == c.Etag { return true, nil } body := cloudwire.Tiles{TileID: c.TileID, Etag: c.Etag, Res: c.Res, StartMs: c.StartMs, StepMs: c.StepMs, Series: c.Series, Data: c.Data} - sent, err := w.sendTile(snap, ts, &body) + sent, err := w.sendTile(ts, &body) if sent { ts.have[c.TileID] = c.Etag } return sent, err } -// pack reads [from, to) of the tile at start from the ledger and packs the -// tile as the app protocol does, so a closed tile is the same bytes as over -// Noise. Buckets from to on are left out: they have not closed. -func (w *writer) pack(hs HistorySite, ts *tileState, start, from, to int64) (protocol.HistChunk, error) { - l := w.link +// PackTile reads [from, to) of the tile at start through hs and packs the +// tile as the app protocol does, so a closed tile is the same bytes as the +// box serves over Noise. Buckets from to on are left out: they have not +// closed. +func PackTile(ctx context.Context, hs HistorySite, start, from, to int64) (protocol.HistChunk, error) { res := cloudwire.TileRes - ctx, cancel := context.WithTimeout(context.Background(), tilesReadTimeout) - defer cancel() windows, err := hs.HistoryWindow(ctx, cloudwire.TileSeries, protocol.ResolutionStepMs[res], from, to) if err != nil { - l.log.Debug("FTW Cloud tiles: the ledger did not answer", "err", err) - ts.retryAt = time.Now().Add(tilesRetryAfter) return protocol.HistChunk{}, err } for name, samples := range windows { @@ -196,9 +210,27 @@ func (w *writer) pack(hs HistorySite, ts *tileState, start, from, to int64) (pro return protocol.PackTile(res, 1, start, cloudwire.TileSeries, windows, to), nil } -// sendTile sends one tile under the byte budget. Past it, tiles wait a -// minute and go again from where they were. -func (w *writer) sendTile(snap grantSnapshot, ts *tileState, body *cloudwire.Tiles) (bool, error) { +// pack is PackTile under the read timeout. A ledger that does not answer +// holds tiles back a minute. +func (w *writer) pack(hs HistorySite, ts *tileState, start, from, to int64) (protocol.HistChunk, error) { + ctx, cancel := context.WithTimeout(context.Background(), tilesReadTimeout) + defer cancel() + c, err := PackTile(ctx, hs, start, from, to) + if err != nil { + w.link.log.Debug("FTW Cloud tiles: the ledger did not answer", "err", err) + ts.retryAt = time.Now().Add(tilesRetryAfter) + } + return c, err +} + +// sendTile sends one tile under the byte budget, if the grant in force +// still allows tiles: a revoke may land while the ledger is read. Past the +// budget, tiles wait a minute and go again from where they were. +func (w *writer) sendTile(ts *tileState, body *cloudwire.Tiles) (bool, error) { + snap := w.link.current() + if !snap.allowed(cloudwire.MsgTiles) { + return false, nil + } sent, err := w.sendBody(snap, cloudwire.MsgTiles, body, false) if err == nil && !sent { ts.retryAt = time.Now().Add(tilesRetryAfter) diff --git a/go/internal/cloudlink/tiles_test.go b/go/internal/cloudlink/tiles_test.go index 748508bd4..55a888781 100644 --- a/go/internal/cloudlink/tiles_test.go +++ b/go/internal/cloudlink/tiles_test.go @@ -408,3 +408,27 @@ func TestALedgerThatFailsHoldsTilesBackAndNothingElse(t *testing.T) { t.Fatalf("a failing ledger was read %d times within its retry", reads) } } + +// History taken out of the grant and put back: the service dropped what it +// held, so every tile goes again, as at connect, on the same link. +func TestHistoryTakenOutAndPutBackSendsEveryTileAgain(t *testing.T) { + svc := newFakeService(t) + h, _ := newTileHarness(t, svc, nil) + grantHistory(t, h) + waitTiles(t, svc, 5) + conns := svc.conns + + h.grant(t, protocol.ScopeLiveRead) + time.Sleep(300 * time.Millisecond) + if n := len(tilesSent(t, svc)); n != 5 { + t.Fatalf("%d tiles went without history in the grant", n) + } + grantHistory(t, h) + got := waitTiles(t, svc, 10) + if svc.conns != conns { + t.Fatalf("the link dialled again: %d connections", svc.conns) + } + if !got[5].Whole() || !got[5].Partial || got[6].Partial || got[9].TileID != tileID(trailStart-4*tileSpan) { + t.Fatalf("after history came back: %v", tileNames(got[5:])) + } +} diff --git a/go/protocol/cloudwire/tiles.go b/go/protocol/cloudwire/tiles.go index 346c07905..7fc0d77e4 100644 --- a/go/protocol/cloudwire/tiles.go +++ b/go/protocol/cloudwire/tiles.go @@ -51,13 +51,21 @@ const ( TilePiecesMax = 16 // TilesClosedMax is how many closed tiles 48 hours can touch. TilesClosedMax = 5 - // TileSeriesMax caps the columns of one tile, as hist.query does. - TileSeriesMax = 16 + // TileEnvelopeMax bounds one tiles envelope. A tile of the four series + // is about 2.6 KB, so FTW Cloud drops anything larger, padding with keys + // it ignores included. + TileEnvelopeMax = 4 << 10 ) -// TileSeries are the columns the box sends, in this order. They are what the -// app asks for over Noise, so a closed tile is the same bytes either way. -var TileSeries = []string{"grid_w", "pv_w", "battery_w", "load_w"} +// TileSeries are the columns the box sends, in this order, and the only ones +// a tile may carry. They are what the app asks for over Noise, so a closed +// tile is the same bytes either way. +var TileSeries = []string{ + protocol.FieldNames[protocol.FidGridW], + protocol.FieldNames[protocol.FidPvW], + protocol.FieldNames[protocol.FidBatteryW], + protocol.FieldNames[protocol.FidLoadW], +} // Tiles is one tile, or one piece of the trailing tile. type Tiles struct { @@ -127,15 +135,8 @@ func (t Tiles) Check() error { if t.TileID != protocol.TileIDFor(t.Res, 1, t.TileStart()) { return bad("tile id does not name the tile its start is in") } - if len(t.Series) == 0 || len(t.Series) > TileSeriesMax { - return bad("series count") - } - seen := map[string]bool{} - for _, s := range t.Series { - if s == "" || len(s) > 64 || seen[s] { - return bad("series names") - } - seen[s] = true + if !slices.Equal(t.Series, TileSeries) { + return bad("series are not TileSeries") } n := t.Points() if n < 1 || len(t.Data) != n*4*len(t.Series) { diff --git a/go/protocol/cloudwire/tiles_test.go b/go/protocol/cloudwire/tiles_test.go index 57e22565b..23621ce23 100644 --- a/go/protocol/cloudwire/tiles_test.go +++ b/go/protocol/cloudwire/tiles_test.go @@ -109,6 +109,23 @@ func TestTheThreeShapesCheck(t *testing.T) { // A closed tile is what the box serves over Noise for the same series: the // same id, etag and bytes. +// The series are the registry's names, and a whole tile fits the bound +// FTW Cloud holds a tiles envelope to. +func TestTileSeriesAreTheRegistrysAndATileFitsItsBound(t *testing.T) { + if !slices.Equal(TileSeries, []string{"grid_w", "pv_w", "battery_w", "load_w"}) { + t.Fatalf("TileSeries %v", TileSeries) + } + b := vectorTiles()[0].body + env, err := Seal(nil, "home", MsgTiles, b) + if err != nil { + t.Fatal(err) + } + raw, _ := env.Encode() + if len(raw)+64+6 > TileEnvelopeMax { + t.Fatalf("a closed tile with its signature is %d bytes", len(raw)+64) + } +} + func TestAClosedTileIsTheNoiseTile(t *testing.T) { noise := protocol.PackTile(TileRes, 1, vectorTilePrev, TileSeries, vectorWindows(), vectorTileStart+3_600_000) got := vectorTiles()[0].body.Chunk() @@ -131,6 +148,14 @@ func TestCheckRefusesWhatIsNotATile(t *testing.T) { b.Etag = protocol.EtagOf(b.Data) }, "no series": func(b *Tiles) { b.Series = nil }, + "another series": func(b *Tiles) { + b.Series = []string{"grid_w", "pv_w", "battery_w", "ev_w"} + }, + "one series less": func(b *Tiles) { + b.Series = b.Series[:3] + b.Data = b.Data[:3*b.Points()*4] + b.Etag = protocol.EtagOf(b.Data) + }, "twice the same series": func(b *Tiles) { b.Series = []string{"grid_w", "grid_w", "pv_w", "load_w"} },