diff --git a/Cargo.lock b/Cargo.lock index 2b190ce3..27a074bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -883,8 +883,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7b4fbfd0a8b397d0dc1a0dc7308308242911147224c4e1a56130168e7ad48f" +source = "git+https://github.com/WithAutonomi/ant-protocol?rev=d11d1010d93958fee6e63e4aace8770f39a0678b#d11d1010d93958fee6e63e4aace8770f39a0678b" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 2e8359d3..03efec06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,6 +228,13 @@ webrtc-direct = [ "dep:self_encryption", ] +[patch.crates-io] +# Pointers (ADR-0016) add the `Pointer` record and its wire messages on top of +# the published 3.0.0, so this is the only entry that has to leave the release +# baseline. A rev, not a branch, so the pin is immutable. Drop it once the +# pointer PR lands and a release includes it. Versions are bumped at release. +ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "d11d1010d93958fee6e63e4aace8770f39a0678b" } + [profile.release] lto = true codegen-units = 1 diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md new file mode 100644 index 00000000..ac1a1909 --- /dev/null +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -0,0 +1,385 @@ +# ADR-0016: Pointers — paid mutable references with an immutable owner + +- **Status:** Proposed +- **Date:** 2026-09-18 +- **Decision owners:** Anselme (@grumbach) +- **Related:** ADR-0002 (audit), ADR-0004 (commitment-bound pricing), ADR-0008 (per-record pricing), ADR-0009 (audit families), ADR-0011 (capacity-gated discovery), ADR-0014 (file store), ADR-0015 (browser clients) + +## Context + +2.0 stores only immutable chunks. 1.0's pointer had three defects: updates +after the first were free, the merge rule diverged permanently on equal +counters, and the address *was* the owner's key, so ownership could never +change. + +We fix the first two and keep the third deliberately. A former owner keeps its +key forever, so transferable ownership cannot be made fork-proof by any local +rule: hash tie-breaks are grindable in ~2 keygens and payment-order ties fall to +a pre-buy. Declining transfer is what lets this design be small enough to trust. + +## Decision + +One record. No genesis object, no certificates, no lineage. + +```rust +pub struct Pointer { // 5,303 bytes + version: u8, // 1 + owner: MlDsa65PublicKey, // 1,952 — the identity; the address derives from it + counter: u64, // 8 — orders states; larger wins + target: PointerTarget, // 33 — kind tag + address; opaque to a node + sig: MlDsa65Signature, // 3,309 — over every field above +} +``` + +Five fields and nothing else — no cached bytes, no cached identifiers. Encoding +is fixed-width, big-endian and hand-rolled with no serde, so there is exactly +one byte sequence for a record and re-encoding is always identical to what was +signed. The address and `state_id` are derived from fields +already present, so they are computed rather than stored. + +The key is carried because it has to be: ML-DSA has no key recovery and a +1,952-byte key cannot be a 32-byte address. That is the whole reason a pointer +is 5,303 bytes rather than ~3,350, and the price of validating one with no +fetch. + +### Two identities + +```text +A = BLAKE3::derive_key("autonomi.pointer.address.v1", owner) routes +state_id = BLAKE3::derive_key("autonomi.pointer.state.v1", body) authorizes payment +``` + +Derive-key, not a hash of a prefix. A chunk's address is `BLAKE3(content)`, so a +prefix separates nothing: a chunk holding the prefix and an owner key would land +on exactly that owner's address, letting anyone squat an address before its +owner used it, and — since `state_id` is what a pointer's storage is paid +against — letting one settled quote buy both a pointer and a chunk. Derive-key +is a different BLAKE3 mode. Both still produce 32 bytes and the ranges are not +disjoint; what changed is the cost. Landing a chunk on a pointer identity now +means finding a preimage under one mode for an output of the other, which is +the security assumption BLAKE3 is built on, rather than a string anyone can +write down. + +**Public-key addressed and self-verifying.** `A` is a pure function of the owner +key, and the key is in the record, so a node validates a pointer from its own +bytes: one hash, one signature check, no fetch. + +`A` and `state_id` are separate because `A` is stable for the pointer's life +while the paid identifier must change with every update — paying against `A` +would make every update after the first free. + +### Pay to create, pay to update + +Every stored state is paid against its own `state_id`, so **one payment buys +one state**. Creating and updating are the same operation: a node takes any +paid record that beats what it holds under the merge rule, whatever its counter. +The client creates at 0 and updates by signing one past the counter the network +serves, but nothing requires exactly one. + +The counter orders states; it does not meter them. A number that is skipped is +never stored, so skipping avoids no payment that was owed. And requiring `+1` +would break catching up, since nothing replicates a pointer. A write ends once +its quorum has answered — five of the seven peers — so a peer can miss an +update, and one that joins the group later holds nothing at all. Under a `+1` +rule either would refuse every later update for good, and three such peers +would leave no write able to reach its quorum. Under the merge rule each takes +the next update, however far ahead of it that is. + +An owner who jumps straight to `u64::MAX` limits only themself, and does not +even freeze the pointer: equal counters still resolve by target, below. + +### Merge + +```text +1. larger counter +2. smaller target bytes +``` + +A total order on the states of **one address**. Records of different owners are +not comparable and never contend. **Equal state never replaces**: ML-DSA signing +is randomized, so one state has unboundedly many valid encodings, and ordering +record *bytes* would let an owner sign one paid state repeatedly and have every +submission win. + +### Validation order + +```text +length → version → structure → compare with held → admission → signature → payment → commit +``` + +Cheap first. A resubmission of what is held is refused before any signature +check. Admission (capacity, responsibility for `A`) precedes the signature, so a +forged record for someone else's address buys no cryptography. "Compare with +held" reads the held record back rather than trusting the index, so no answer +describes a record the node cannot serve. The commit re-checks under its lock, +because a newer state can land while payment verifies. Every step off the async +executor: the read, the signature check and the write each run on a blocking +thread, so a flood of arrivals cannot occupy the runtime's workers. + +### Replication + +Pointers replicate through the same engine chunks do — the same close groups, +neighbour-sync rounds, churn triggers, quorum, pruning and possession rules — +but by **state**, not by key. The chunk pipeline assumes a record never changes +and that its key is the hash of its bytes; a pointer's address is stable while +its state changes, and two honest replicas may hold different valid signatures +over one state. + +- **Fresh.** A node that accepts a paid state from a client forwards the record, + with the proof that paid for it, to the rest of the close group. Each receiver + checks the signature, its own responsibility (across the paid width, as for a + chunk offer) and the payment itself before storing. So a paid state that + reached one honest node reaches the whole group, whichever members the client + wrote to. +- **Repair.** Every neighbour-sync round pushes hints — the states the sender + holds that the receiver should hold — to the peers being synced, and a peer + that syncs with a node gets that node's hints back. A receiver that lacks a + hinted state, or holds an older one, asks the close group which state each + holds, adopts the best state a quorum of them hold **exactly**, and fetches it + from one of them. The record verifies itself; the quorum stands in for the + payment proof, as presence quorum does for a chunk. The quorum is the one a + chunk needs, counted over the whole close group: a peer that cannot be asked + counts as unanswered, never as a vote. This is how a node that missed an + update, joined late, or lost a record across a restart is brought level. +- **Pruning.** A record the node has been outside the retention width of for the + hysteresis period is deleted — at once if the node is outside a complete + paid-width group, otherwise only once all but one of the current close group + prove they hold that state or a newer one by returning a valid record. A proof + is a record, not a claim: signatures are checked. +- **Possession.** Some minutes after offering a fresh state, the offering node + asks each member for the record. One that is still responsible and cannot + produce that state or a newer one is penalised, as a chunk holder is. + +Six messages carry this, appended to the replication enum so every earlier +discriminant keeps its value: a fresh offer and a hint push (one-way), and a +fetch and a state query with their responses. An older peer cannot decode them. +One-way pushes to it are simply lost, and requests only ever go to peers that +have sent a pointer message themselves — a hint push goes out every round, empty +or not, so capability is learned within a cycle — so an older peer is never asked +something it cannot answer and never penalised for its silence. + +### Storage commitments and audits + +A node commits to the pointers it is responsible for in the same signed storage +commitment as its chunks (ADR-0002, ADR-0004), so they are priced and audited as +chunks are. + +- **Leaf.** A chunk is committed as `(key, key)`, since its address is the hash + of its bytes. A pointer's address is not, and its bytes change with every + update, so it is committed as `(A, pointer_leaf_hash(A))`, a BLAKE3 derive-key + of the address under its own context. The root binds which pointers a node + holds, never their current state, so an update does not move it and no audit + can fail an honest holder for having taken one mid-audit. +- **Price.** A quote is priced from the key count of the commitment it pins, so + every committed pointer counts toward it exactly as a chunk does. +- **Round 1.** A pointer leaf is reported at its address with that hash, the + fixed record length, and a nonced root over the bytes of the record the node + holds, under the audit's fresh nonce, as a chunk leaf's is. So the node has to + read every pointer in the audited subtree before it learns which few will be + sampled. The auditor accepts that one leaf shape besides `(key, key)`, and only + at exactly a pointer's length, so it cannot stand in for a chunk of any other + size. A node that has lost a committed pointer refuses round 1, which is a + confirmed failure, as a lost chunk is. +- **Round 2.** Where a chunk is proved by a Bao slice and a nonced opening, a + pointer is proved by its whole signed record. The auditor checks the + signature, that the record belongs at the committed address, and that its + nonced root is the one round 1 gave. A peer signs its own commitment, so + without the signature check it could commit any key under the hash of cheap + bytes it holds; without the nonced root it could hold nothing and fetch the + sampled records on demand. At most five records are opened, a few tens of + kilobytes, well under the audit message ceiling. +- **Updates between the rounds.** The owner may update a pointer after round 1 + bound it. The store keeps the record an update replaced for five minutes, + longer than an audit session lives, and round 2 serves it beside the new one; + the auditor accepts whichever matches. Two updates to one pointer inside the + same audit would fail an honest holder, and would take the owner two paid + updates within seconds of each other. +- **Retention and pruning.** A retained commitment that still holds a pointer + vetoes its deletion, as for a chunk, so a peer pinning that commitment can + still audit it. The persisted retention keeps which leaves are pointers in a + file beside it, keyed by commitment hash, so a restart rebuilds the exact + signed root. The retention file itself is unchanged, so a node rolled back to + an earlier release still reloads every commitment it can answer for, and + drops only those that commit a pointer. + +Round 2 carries a new slice item, so the subtree audit's protocol id moves to +`v2`, as ADR-0009 did before. Nodes on different ids do not audit each other +across the upgrade. A round-1 request that goes unanswered costs the auditor the +bounded trust penalty ADR-0009 accepted; nothing is misjudged as missing data. + +Records are kept one file each under `{root}/pointers//`, 256 shards by +the address's last byte as the chunk store keeps them. Opening the store parses +each record's structure but does not verify its signature; every record was +verified when it was committed and every read verifies it again. + +## What this defends against + +| Attack | Defence | +|---|---| +| Tamper with any byte | Signature over the whole body | +| Swap the owner key | `A` is derived from it; the record no longer belongs at its address | +| Store at someone else's address | Same | +| Fork / equivocate at one counter | Total order on `(counter, target)`: every node given the same records picks the same one, a read merges the close group's answers rather than trusting the first, and replication gives every member the states that reached a quorum, so the group converges | +| Replay an older record | Loses on counter | +| Re-sign one paid state N times | Equal state never replaces; nothing is written | +| Pay once, jump the counter | Nothing to defend: one payment stores one state whatever its counter, and a skipped number is never stored | +| Pay for a chunk to fund a pointer | A quote signs its content, not the record kind, so the defence is that the content cannot be shared: putting a chunk on a `state_id` means breaking BLAKE3 across its two modes. The paid cache is keyed by a typed `Chunk` vs `Pointer` target as well, so the two never alias even in memory, whatever the bytes | +| Merkle proof with no issuer check | Refused for pointers; single-node proofs only | +| Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | +| Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | +| Collide a pointer and a chunk address | Takes a cross-mode BLAKE3 break, and is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | +| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | +| One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | +| Two peers decide it | **Not defended against, at the read.** Two colluding close-group peers clear a read's bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Replication does not spread such a state: the rest of the group adopts only what a quorum of it holds, so the honest members keep the paid state, but a reader that happens to hear from both colluders still sees theirs. Raising the read's bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold | +| Commit to pointers it does not hold, for price or audit credit | A committed pointer is proved in round 2 by the whole signed record, which the auditor verifies and checks belongs at the address; the pointer leaf shape is accepted only at the fixed record length | +| Relay pointers instead of storing them | Round 1 binds a nonced root over the bytes of every pointer in the audited subtree before the sample is drawn, and round 2 must reproduce one. Another replica's copy of the same state does not match: every signature is randomised. Still weaker than for a chunk by nature: a record is 5 KB, so fetching a whole subtree of them on demand costs a relay far less than a subtree of chunks would | +| Get a group to adopt a state nobody paid for | Repair adopts only a state a quorum of the close group hold exactly, counted over the whole group, and a fresh offer is stored only after the receiver verifies its payment itself | +| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | + +## Consequences + +- Validating a pointer needs nothing but the pointer — no quorum, no lineage, no + Sybil exposure in ownership. +- Creation is one record and one payment, with no retention dependency. +- A pointer write must reach one more peer than a chunk write does. A chunk is + self-proving, so one copy settles it; a pointer read has to decide which of + several signed states is current, and that answer has to come from more than + one peer. +- **Ownership cannot change.** Handover is indirection: point at a new pointer + the recipient owns. The old owner keeps write access forever, so it is a + revocable forwarding state, not a sale. +- **Key compromise is permanent.** No rotation, no recovery. +- The inlined key costs 1,952 bytes on every read, forever — a deliberate trade + for self-contained validation. +- Determinism is not freshness: an eclipsed reader can be handed an older, + correctly signed value and cannot tell. +- Replicas may hold different valid signatures of one state; nothing compares + record bytes across replicas. +- A pointer audit opens a whole 5,303-byte record, and up to two of them after + an update, where a chunk audit opens one 1 KiB block, so a round 2 over + pointers is several times larger. It stays bounded by the five-leaf cap. +- A node keeps the record each update replaced in memory for five minutes, up + to 2,048 of them, about 11 MB. +- The subtree audit's protocol id moved to `v2`, so across the upgrade the + old and new releases do not audit each other. + +## Implementation status + +Built: the record and wire messages (`ant-protocol`); the store with +merge-on-put; request dispatch; payment routed at `state_id` with the close +group of `A`; admission gates; cross-kind refusal; per-request latency events +beside the chunk ones (`pointer_put_rpc`, `pointer_get_rpc`); and the client — +create, update, quorum store, merged reads and chain resolution, a write that +falls short retried with the proof it already paid for, a split payment for an +external signer, and the `ant pointer` commands. + +**Built: replication** (see Replication above): fresh offers with payment, +neighbour-sync repair by quorum over exact states, pruning with possession +proofs, and post-offer possession checks, wired into the engine's sync rounds, +churn triggers and cycle completion. A node that missed an update, joined late, +or lost a record is brought level by the next sync round rather than the next +write. + +**Built: commitments and audits** (see Storage commitments and audits above): +responsible pointers are leaves of the storage commitment, count toward the +quoted price, are spot-checked by the subtree audit in both rounds, survive a +restart in the persisted retention, and are kept from pruning while a retained +commitment holds them. + +**Built: browser clients** (ADR-0015). Each of the four decisions at that +security boundary is made explicitly: + +- **Admission.** The WebRTC-direct listener admits a pointer GET and a paid + pointer PUT through `chunk_protocol`, each bounded by the small response size + a quote gets. One record is 5,303 bytes, well inside it. +- **Sanitizing.** Replies pass the sanitizer. A pointer outcome names only an + address and a state identifier, and a record is owner-signed public data. An + error is redacted as for a chunk, and a refused payment is reported without + its detail. +- **Classification.** A pointer PUT is a paid write: it takes the data lane, + holds its connection exclusively as a chunk PUT does, and goes only to nodes + on the page's payment network. A pointer GET returns one small record, so it + stays on the RPC lane, like a quote, and needs no bulk read slot. +- **Rules.** The browser runs the native client's own read quorum, + corroboration, write quorum and chain resolution. Only the wallet is the + page's. + +A node advertises `pointer_protocol` in HELLO when it admits pointers and has a +pointer store to serve them from. A browser +client never sends a pointer request to a node that does not, so an older node's +refusal is never mistaken for a failed peer. The owner key crosses into the page +as its 32-byte FIPS 204 seed. A pointer payment is one quote and, unlike a file +upload, keeps no recovery journal: a payment interrupted between broadcast and +receipt is paid again on retry. + +## Validation + +- Every delivery order of a record set converges to one value, exhaustively over + all permutations, including on a node started empty and one restarted. +- 64 valid signatures over one paid state yield one stored record and one file. +- A resubmission and a stale arrival are both refused before any signature check. +- Any paid record that beats the held one is taken, including a first record + above counter 0 and a jump to `u64::MAX`; anything older is refused as stale, + and the client cannot wrap a terminal counter back to 0. +- A fork at one counter held by two nodes is healed on both by any later counter. +- All 256 `version` values give distinct paid identifiers. +- Golden vectors pin the encoding, both identities and the signing context. +- A crafted chunk cannot satisfy a pointer's paid-cache entry. +- An acknowledgement naming a different address or state is refused, and a read + keeps the winner whatever order the replies arrive in. +- A write ends as soon as its quorum acknowledges, so one unreachable peer + cannot stall it. A read ends when a quorum has answered *and* one of the + states they named has the backing a read demands; short of that it keeps + asking, and if the group is exhausted without either, it reports a shortfall + rather than presenting one peer's word as the network's answer. +- A state only one peer named does not suppress the state the others agree on, + in any arrival order. +- The write and read thresholds overlap in at least the two peers a read + demands, at every group width from 1 to 64. +- A resubmission repairs a record whose file the disk lost — at any counter, + not just at creation — rather than being acknowledged as unchanged. A file + swapped for a different valid record is not answered for either. +- Two nodes given the same two paid states in opposite orders keep the same + record, through the request handler and its admission gate rather than the + store alone. +- The chunk preimages the old prefix construction handed out no longer land on + either identity. (No test can say more: that no content does is the preimage + assumption, not a property one can check.) +- Replication across a live multi-node network: a paid PUT to one node reaches + its whole close group; an update replaces the old state everywhere; a node that + missed an update is repaired by neighbour sync alone; a node that joins later + obtains existing pointers through the engine's own loops; a state only one node + holds is not adopted, though its hints arrived; an unpaid fresh offer is + refused; the possession check penalises only the member that dropped the + record; pruning deletes only once the close group proves it holds the record, + and a node far outside the group prunes without asking. +- From a browser: a node admits pointer requests at its WebRTC boundary, bounds + a full record inside the small response limit, and passes every pointer reply + through the sanitizer with errors redacted. In real Chromium against a live + local network with on-chain payment, a pointer is created, updated, pointed + at a second pointer, read back by a client that wrote nothing and resolved + down the chain to its chunk. +- Storage audits, through the live responders and judged by the auditor's own + checks: a node holding committed pointers and chunks passes both rounds; an + update between the rounds does not fail it, and does if the replaced record + is not kept; a node that lost a committed pointer fails round 1, and one that + loses it after round 1 is caught in round 2; another owner's valid record is + not proof of the committed one; a relay that answers round 1 without the + bytes fails, as does one serving another replica's copy of the same state; a + pointer leaf at any other length is refused. Over a live network, a node holding its + committed pointers passes the audit and one that dropped them fails it. +- A commitment holding pointers survives a restart with its exact pin. An + older release reading the same retention keeps the chunk-only commitment a + peer pinned before the upgrade, and a pointer-leaves file from another + snapshot attaches nothing. +- Pruning keeps a pointer a retained commitment holds, and drops it once none + does. +- Fresh offers, hints, fetches and state queries are covered by the replication + protocol's per-variant tests: family, size ceiling, round-trip. +- End to end against a live testnet with real settlement: create, update, read + back, resolve a chain to its chunk, repeat a stored state, read an address + nobody wrote. Every close-group node answers a paid update to a pointer it + already holds with success, not a refusal. A record that skips counters is + taken. Nodes that missed updates take the next one and the group converges. + A fork at one counter across the group reads as its merge winner, and one + update at the next counter heals it on every node. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8fc52bc4..5061984e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,3 +35,4 @@ See [`TOOLING.md`](./TOOLING.md) for `adrs`, `adr-kit`, and AI harness setup. - [ADR-0014: One File Per Chunk, and Retiring LMDB Without Losing Data](./ADR-0014-file-based-chunk-store-and-lmdb-retirement.md) - [ADR-0013: Settlement version and pre-payment compatibility](./ADR-0013-settlement-version-and-pre-payment-compatibility.md) - [ADR-0015: Direct browser clients over WebRTC Direct](./ADR-0015-direct-browser-clients-over-webrtc-direct.md) +- [ADR-0016: Pointers — paid mutable references with an immutable owner](./ADR-0016-pointers-immutable-owner.md) diff --git a/src/devnet.rs b/src/devnet.rs index 8a32e088..b6f75bed 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -973,11 +973,19 @@ impl Devnet { let storage = Arc::new(storage); let payment_verifier = Arc::new(payment_verifier); - Ok(AntProtocol::new( - storage, - payment_verifier, - Arc::new(quote_generator), - )) + // Same pointer wiring as a production node, so a devnet exercises the + // real path rather than a node that silently refuses every pointer. + let pointer_store = crate::pointer::PointerStore::new(storage.root_dir()) + .await + .map_err(|e| DevnetError::Startup(format!("Failed to open pointer store: {e}")))?; + let pointers = crate::pointer::PointerService::new(pointer_store) + .with_chunk_store(Arc::clone(&storage)) + .with_payments(Arc::clone(&payment_verifier)); + + Ok( + AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers), + ) } #[allow(clippy::too_many_lines)] diff --git a/src/error.rs b/src/error.rs index f71ed7ed..34a5e002 100644 --- a/src/error.rs +++ b/src/error.rs @@ -60,3 +60,20 @@ pub enum Error { #[error("node is shutting down")] ShuttingDown, } + +impl From for Error { + /// Map a wire-level pointer rejection onto the node's error type. + /// + /// Signature failures become [`Error::Crypto`] and everything else becomes + /// [`Error::Protocol`], so a caller can still tell "these bytes are not a + /// pointer" from "these bytes are not signed by the key they carry". + fn from(error: ant_protocol::pointer::PointerError) -> Self { + use ant_protocol::pointer::PointerError; + match error { + PointerError::SignatureInvalid | PointerError::SigningFailed(_) => { + Self::Crypto(error.to_string()) + } + other => Self::Protocol(other.to_string()), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 2af24b0c..852c837a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,8 +17,10 @@ //! //! ## Data Types //! -//! Currently supports a single data type: +//! Two data types: //! - **Chunk**: Immutable content-addressed data (hash(value) == key) +//! - **Pointer**: A paid mutable reference signed by an immutable owner, stored +//! at an address derived from the owner key (see [`mod@pointer`] and ADR-0016) //! //! ## Example //! @@ -52,6 +54,7 @@ pub mod event; pub mod logging; pub mod node; pub mod payment; +pub mod pointer; pub mod replication; pub mod storage; pub mod upgrade; @@ -77,6 +80,9 @@ pub use error::{Error, Result}; pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; +pub use pointer::{ + Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, PutOutcome, +}; pub use replication::{config::ReplicationConfig, ReplicationEngine}; pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; diff --git a/src/node.rs b/src/node.rs index 1c640cb5..56813244 100644 --- a/src/node.rs +++ b/src/node.rs @@ -238,7 +238,7 @@ impl NodeBuilder { fresh_rx: UnboundedReceiver, shutdown: &CancellationToken, ) -> Result<(Option, Option>)> { - let engine = match ReplicationEngine::new( + let mut engine = match ReplicationEngine::new( repl_config, Arc::clone(p2p), protocol.storage(), @@ -271,6 +271,14 @@ impl NodeBuilder { } }; + // ADR-0016: pointers replicate through the same engine. The PUT handler + // hands each newly stored paid state to it on this channel. + if let Some(service) = protocol.pointer_service() { + let (writes, fresh_writes) = tokio::sync::mpsc::unbounded_channel(); + service.attach_fresh_writes(writes); + engine.with_pointers(service.store().clone(), fresh_writes); + } + // ADR-0004: wire the engine's commitment state as the quote generator's // commitment source so quotes force their price from the live storage // commitment. Done here because the engine owns the commitment state and is @@ -576,7 +584,21 @@ impl NodeBuilder { let storage = Arc::new(storage); let payment_verifier = Arc::new(payment_verifier); - let protocol = AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)); + // Pointers live beside the chunks, under the same root. Opening the + // store here is what makes pointer PUT/GET answerable at all: without + // it every pointer request is refused, which is the right answer for a + // node that keeps none but the wrong one for a node that should. + let pointer_store = crate::pointer::PointerStore::new(&config.root_dir).await?; + let pointers = crate::pointer::PointerService::new(pointer_store) + // Refuse a pointer whose address a chunk already occupies, rather + // than letting one kind silently overwrite the other. + .with_chunk_store(Arc::clone(&storage)) + // Payment is verified against each record's state, so every update + // is paid for rather than riding the first one. + .with_payments(Arc::clone(&payment_verifier)); + + let protocol = AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers); info!( "ANT protocol handler initialized with ML-DSA-65 signing (protocol={CHUNK_PROTOCOL_ID})" diff --git a/src/payment/cache.rs b/src/payment/cache.rs index 174c45b8..857482cf 100644 --- a/src/payment/cache.rs +++ b/src/payment/cache.rs @@ -10,6 +10,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; pub use super::quote::XorName; +pub use super::verifier::PaymentTarget as PaidKey; /// Default cache capacity (100,000 entries = 3.2MB memory). const DEFAULT_CACHE_CAPACITY: usize = 100_000; @@ -25,7 +26,7 @@ const DEFAULT_CACHE_CAPACITY: usize = 100_000; /// entries satisfy weaker lookups. #[derive(Clone)] pub struct VerifiedCache { - inner: Arc>>, + inner: Arc>>, hits: Arc, misses: Arc, additions: Arc, @@ -101,8 +102,8 @@ impl VerifiedCache { /// Returns `true` if the `XorName` is cached (verified to exist on autonomi). /// Paid-list and client-PUT lookups must use their stricter helpers. #[must_use] - pub fn contains(&self, xorname: &XorName) -> bool { - let found = self.inner.lock().get(xorname).is_some(); + pub fn contains_key(&self, key: &PaidKey) -> bool { + let found = self.inner.lock().get(key).is_some(); if found { self.hits.fetch_add(1, Ordering::Relaxed); @@ -119,11 +120,11 @@ impl VerifiedCache { /// A client-PUT entry returns `true` here because it passed the stricter /// store-admission path at the caller. #[must_use] - pub fn contains_paid_list_verified(&self, xorname: &XorName) -> bool { + pub fn contains_paid_list_verified_key(&self, key: &PaidKey) -> bool { let found = self .inner .lock() - .get(xorname) + .get(key) .copied() .is_some_and(|level| level.satisfies(VerificationLevel::PaidList)); @@ -142,11 +143,11 @@ impl VerifiedCache { /// Paid-list entries return `false` here because they did not pass the /// client-PUT store-admission path. #[must_use] - pub fn contains_client_put_verified(&self, xorname: &XorName) -> bool { + pub fn contains_client_put_verified_key(&self, key: &PaidKey) -> bool { let found = self .inner .lock() - .get(xorname) + .get(key) .copied() .is_some_and(|level| level.satisfies(VerificationLevel::ClientPut)); @@ -163,30 +164,30 @@ impl VerifiedCache { /// /// This should be called after verifying that data exists on the autonomi network. /// Also upgrades an existing paid-list-verified entry. - pub fn insert(&self, xorname: XorName) { - self.insert_with_level(xorname, VerificationLevel::ClientPut); + pub fn insert_key(&self, key: PaidKey) { + self.insert_with_level(key, VerificationLevel::ClientPut); } /// Add a `XorName` verified under paid-list admission checks. /// /// Never downgrades an existing client-PUT-verified entry. - pub fn insert_paid_list_verified(&self, xorname: XorName) { - self.insert_with_level(xorname, VerificationLevel::PaidList); + pub fn insert_paid_list_verified_key(&self, key: PaidKey) { + self.insert_with_level(key, VerificationLevel::PaidList); } - fn insert_with_level(&self, xorname: XorName, level: VerificationLevel) { + fn insert_with_level(&self, key: PaidKey, level: VerificationLevel) { let added = { let mut inner = self.inner.lock(); // `get_mut` refreshes LRU recency for existing entries of either kind. - if inner.get(&xorname).is_some() { - if let Some(existing) = inner.get_mut(&xorname) { + if inner.get(&key).is_some() { + if let Some(existing) = inner.get_mut(&key) { if !existing.satisfies(level) { *existing = level; } } false } else { - inner.put(xorname, level); + inner.put(key, level); true } }; @@ -195,6 +196,34 @@ impl VerifiedCache { } } + /// As [`Self::contains_key`], for a chunk at `address`. + #[must_use] + pub fn contains(&self, address: &XorName) -> bool { + self.contains_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::contains_paid_list_verified_key`], for a chunk at `address`. + #[must_use] + pub fn contains_paid_list_verified(&self, address: &XorName) -> bool { + self.contains_paid_list_verified_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::contains_client_put_verified_key`], for a chunk at `address`. + #[must_use] + pub fn contains_client_put_verified(&self, address: &XorName) -> bool { + self.contains_client_put_verified_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::insert_key`], for a chunk at `address`. + pub fn insert(&self, address: XorName) { + self.insert_key(PaidKey::Chunk(address)); + } + + /// As [`Self::insert_paid_list_verified_key`], for a chunk at `address`. + pub fn insert_paid_list_verified(&self, address: XorName) { + self.insert_paid_list_verified_key(PaidKey::Chunk(address)); + } + /// Get current cache statistics. #[must_use] pub fn stats(&self) -> CacheStats { @@ -238,23 +267,23 @@ mod tests { fn test_cache_basic_operations() { let cache = VerifiedCache::new(); - let xorname1 = [1u8; 32]; - let xorname2 = [2u8; 32]; + let key1 = [1u8; 32]; + let key2 = [2u8; 32]; // Initially empty assert!(cache.is_empty()); - assert!(!cache.contains(&xorname1)); + assert!(!cache.contains(&key1)); // Insert and check - cache.insert(xorname1); - assert!(cache.contains(&xorname1)); - assert!(!cache.contains(&xorname2)); + cache.insert(key1); + assert!(cache.contains(&key1)); + assert!(!cache.contains(&key2)); assert_eq!(cache.len(), 1); // Insert another - cache.insert(xorname2); - assert!(cache.contains(&xorname1)); - assert!(cache.contains(&xorname2)); + cache.insert(key2); + assert!(cache.contains(&key1)); + assert!(cache.contains(&key2)); assert_eq!(cache.len(), 2); } @@ -284,21 +313,21 @@ mod tests { #[test] fn test_cache_stats() { let cache = VerifiedCache::new(); - let xorname = [1u8; 32]; + let key = [1u8; 32]; // Miss - assert!(!cache.contains(&xorname)); + assert!(!cache.contains(&key)); let stats = cache.stats(); assert_eq!(stats.misses, 1); assert_eq!(stats.hits, 0); // Add - cache.insert(xorname); + cache.insert(key); let stats = cache.stats(); assert_eq!(stats.additions, 1); // Hit - assert!(cache.contains(&xorname)); + assert!(cache.contains(&key)); let stats = cache.stats(); assert_eq!(stats.hits, 1); assert_eq!(stats.misses, 1); @@ -312,19 +341,19 @@ mod tests { // Small cache for testing eviction let cache = VerifiedCache::with_capacity(2); - let xorname1 = [1u8; 32]; - let xorname2 = [2u8; 32]; - let xorname3 = [3u8; 32]; + let key1 = [1u8; 32]; + let key2 = [2u8; 32]; + let key3 = [3u8; 32]; - cache.insert(xorname1); - cache.insert(xorname2); + cache.insert(key1); + cache.insert(key2); assert_eq!(cache.len(), 2); - // Insert third, should evict xorname1 (least recently used) - cache.insert(xorname3); + // Insert third, should evict key1 (least recently used) + cache.insert(key3); assert_eq!(cache.len(), 2); - assert!(!cache.contains(&xorname1)); // evicted - // Note: after contains call on evicted item, stats will show a miss + assert!(!cache.contains(&key1)); // evicted + // Note: after contains call on evicted item, stats will show a miss } #[test] @@ -409,8 +438,8 @@ mod tests { for i in 0..10u8 { let c = cache.clone(); handles.push(thread::spawn(move || { - let xorname = [i; 32]; - c.insert(xorname); + let key = [i; 32]; + c.insert(key); })); } @@ -418,8 +447,8 @@ mod tests { for i in 0..10u8 { let c = cache.clone(); handles.push(thread::spawn(move || { - let xorname = [i; 32]; - let _ = c.contains(&xorname); + let key = [i; 32]; + let _ = c.contains(&key); })); } diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 7252af43..14925fdf 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -6,7 +6,7 @@ use crate::ant_protocol::CLOSE_GROUP_SIZE; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; -use crate::payment::cache::{CacheStats, VerifiedCache, XorName}; +use crate::payment::cache::{CacheStats, PaidKey, VerifiedCache, XorName}; use crate::payment::pricing::{calculate_price, derive_records_stored_from_price}; use crate::payment::proof::{ deserialize_merkle_proof, deserialize_single_node_proof, detect_proof_type, ProofType, @@ -448,6 +448,65 @@ pub struct PaymentVerifierConfig { pub price_floor: PriceFloorConfig, } +/// What a payment authorizes, and where it routes. +/// +/// One typed value doing both jobs. A chunk pays for its own address; a pointer +/// pays for a *state* while the close group that may quote it is the one around +/// its address — two different addresses with two different meanings. +/// +/// This is also the paid-cache key, and being an enum is what makes that safe: +/// a raw 32-byte key would file both kinds under one value, so anything that +/// put a chunk on a pointer's identifier would buy the pointer's update at +/// chunk price. `state_id` is a `derive_key` output, so putting a chunk on one +/// is a preimage problem rather than a string anyone can write down — and +/// distinct variants cannot collide however the bytes are chosen, so the cache +/// does not rest on that assumption either. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum PaymentTarget { + /// A chunk: paid for and stored at one address. + Chunk(XorName), + /// A pointer state: routed at the pointer's address, paid at its `state_id`. + Pointer { + /// The pointer's address, stable for its life. Selects the close group. + routing: XorName, + /// The state paid for, which changes with every update. + state_id: XorName, + }, +} + +impl PaymentTarget { + /// A chunk, where one address does both jobs. + #[must_use] + pub const fn same(address: XorName) -> Self { + Self::Chunk(address) + } + + /// A pointer, routed at one address and paid at another. + #[must_use] + pub const fn split(routing: XorName, state_id: XorName) -> Self { + Self::Pointer { routing, state_id } + } + + /// The address whose close group is responsible, and whose members' quotes + /// therefore count. + #[must_use] + pub const fn routing(&self) -> &XorName { + match self { + Self::Chunk(address) => address, + Self::Pointer { routing, .. } => routing, + } + } + + /// What the quote must actually name. + #[must_use] + pub const fn content(&self) -> &XorName { + match self { + Self::Chunk(address) => address, + Self::Pointer { state_id, .. } => state_id, + } + } +} + /// The fresh admission path a payment proof is being verified for. /// /// - **`ClientPut`** — the node is admitting a chunk store from a direct @@ -1045,11 +1104,29 @@ impl PaymentVerifier { xorname: &XorName, context: VerificationContext, ) -> PaymentStatus { + self.check_payment_required_keyed(PaymentTarget::Chunk(*xorname), context) + } + + /// As [`Self::check_payment_required`], for an already-typed cache key. + /// + /// A pointer's entry is filed under both its routing address and the state + /// paid for, so it can never be satisfied by a chunk that happens to sit at + /// either value. + #[must_use] + pub fn check_payment_required_keyed( + &self, + key: PaidKey, + context: VerificationContext, + ) -> PaymentStatus { + let xorname = match &key { + PaymentTarget::Chunk(address) => address, + PaymentTarget::Pointer { state_id, .. } => state_id, + }; // Check LRU cache (fast path) let cached = if context.is_store_admission() { - self.cache.contains_client_put_verified(xorname) + self.cache.contains_client_put_verified_key(&key) } else { - self.cache.contains_paid_list_verified(xorname) + self.cache.contains_paid_list_verified_key(&key) }; if cached { if crate::logging::enabled!(crate::logging::Level::DEBUG) { @@ -1099,7 +1176,7 @@ impl PaymentVerifier { let proof_type = payment_proof_type_label(payment_proof); let proof_bytes = payment_proof.map_or(0, <[u8]>::len); let result = self - .verify_payment_inner(xorname, payment_proof, context) + .verify_payment_inner(&PaymentTarget::same(*xorname), payment_proof, context) .await; let elapsed_ms = started.elapsed().as_millis(); @@ -1135,12 +1212,19 @@ impl PaymentVerifier { async fn verify_payment_inner( &self, - xorname: &XorName, + target: &PaymentTarget, payment_proof: Option<&[u8]>, context: VerificationContext, ) -> Result { + // What the proof must name: for a chunk its address, for a pointer the + // state being paid for. Caching under the *address* would mark every + // future update of a pointer as already paid — the 1.0 free-update + // defect — so the cache is keyed separately, and never shares a key + // with the chunk whose address happens to equal this state. + let xorname = target.content(); + let cache_key = *target; // First check if payment is required - let status = self.check_payment_required(xorname, context); + let status = self.check_payment_required_keyed(cache_key, context); match status { PaymentStatus::CachedAsVerified => { @@ -1165,7 +1249,7 @@ impl PaymentVerifier { // Detect proof type from version tag byte match detect_proof_type(proof) { Some(ProofType::Merkle) => { - self.verify_merkle_payment(xorname, proof, context).await?; + self.verify_merkle_payment(target, proof, context).await?; } Some(ProofType::SingleNode) => { let parsed = deserialize_single_node_proof(proof).map_err(|e| { @@ -1180,7 +1264,7 @@ impl PaymentVerifier { } self.verify_evm_payment( - xorname, + target, &parsed.proof_of_payment, &parsed.commitment_sidecars, context, @@ -1208,9 +1292,9 @@ impl PaymentVerifier { // strength. Stronger entries satisfy weaker future lookups, // but not the reverse. if context.is_store_admission() { - self.cache.insert(*xorname); + self.cache.insert_key(cache_key); } else { - self.cache.insert_paid_list_verified(*xorname); + self.cache.insert_paid_list_verified_key(cache_key); } Ok(PaymentStatus::PaymentVerified) @@ -1228,6 +1312,65 @@ impl PaymentVerifier { } } + /// Verify that a pointer state was paid for. + /// + /// `routing_address` is the pointer's address: it selects the close group + /// whose quotes count, and it never changes for the life of the pointer. + /// `paid_content` is the record's `state_id`: it is what the quote must + /// name, and it changes with every update. Passing one address for both — + /// which is all the chunk path can express — would either check the wrong + /// close group or mark every future update of the pointer as already paid. + /// + /// # Errors + /// + /// Returns [`Error::Payment`] if the proof is missing, malformed, names a + /// different state, was issued by a peer outside the routing address's + /// close group, or did not settle on chain. + pub async fn verify_pointer_payment( + &self, + routing_address: &XorName, + paid_content: &XorName, + payment_proof: &[u8], + ) -> Result<()> { + self.verify_pointer_payment_in( + routing_address, + paid_content, + payment_proof, + VerificationContext::ClientPut, + ) + .await + } + + /// As [`Self::verify_pointer_payment`], in `context`. + /// + /// Fresh replication verifies under + /// [`VerificationContext::FreshReplication`], exactly as a chunk offer does: + /// the checks are identical, the context only keeps the price-floor + /// telemetry apart. + /// + /// # Errors + /// + /// As [`Self::verify_pointer_payment`]. + pub async fn verify_pointer_payment_in( + &self, + routing_address: &XorName, + paid_content: &XorName, + payment_proof: &[u8], + context: VerificationContext, + ) -> Result<()> { + let target = PaymentTarget::split(*routing_address, *paid_content); + match self + .verify_payment_inner(&target, Some(payment_proof), context) + .await? + { + PaymentStatus::CachedAsVerified | PaymentStatus::PaymentVerified => Ok(()), + PaymentStatus::PaymentRequired => Err(Error::Payment(format!( + "no settled payment for pointer state {}", + hex::encode(paid_content) + ))), + } + } + /// Get cache statistics. #[must_use] pub fn cache_stats(&self) -> CacheStats { @@ -1250,6 +1393,15 @@ impl PaymentVerifier { self.cache.insert(xorname); } + /// Mark a pointer state as paid, so verifying it needs no chain lookup. + /// The pointer counterpart of [`Self::cache_insert`]: keyed by the state + /// and routed at the address, as a real pointer payment is. + #[cfg(any(test, feature = "test-utils"))] + pub fn cache_insert_pointer(&self, routing_address: XorName, state_id: XorName) { + self.cache + .insert_key(PaymentTarget::split(routing_address, state_id)); + } + /// Mark startup content as prepaid for the in-process browser devnet. /// /// This remains crate-private and feature-gated: it is used only by @@ -1291,11 +1443,15 @@ impl PaymentVerifier { /// was paid 3x. async fn verify_evm_payment( &self, - xorname: &XorName, + target: &PaymentTarget, payment: &ProofOfPayment, commitment_sidecars: &[Vec], context: VerificationContext, ) -> Result<()> { + // `content` is what a quote must name; `routing` is whose close group + // may issue it. For a chunk they are one address, for a pointer they + // are not. + let xorname = target.content(); if crate::logging::enabled!(crate::logging::Level::DEBUG) { let xorname_hex = hex::encode(xorname); let quote_count = payment.peer_quotes.len(); @@ -1332,10 +1488,7 @@ impl PaymentVerifier { for candidate in candidates { let paid_price = candidate.quote.price; let candidate_peer = *candidate.encoded_peer_id.as_bytes(); - match self - .verify_legacy_median_candidate(xorname, candidate) - .await - { + match self.verify_legacy_median_candidate(target, candidate).await { Ok(settled_amount) => { verified_paid_quote = Some((paid_price, settled_amount)); // First settlement-verified median candidate wins the paid @@ -1368,7 +1521,11 @@ impl PaymentVerifier { // unauthenticated bundles can never poison floor telemetry. Shadow // mode logs; enforcement rejects — an economic admission decision // only, never trust/misbehaviour evidence. - self.enforce_price_floor(xorname, paid_price, settled_amount, context) + // The floor compares against the median commitment price of the close + // group that is RESPONSIBLE for the data, so it takes the routing + // address. For a chunk that is the same value; for a pointer the paid + // content is its state, which names no close group at all. + self.enforce_price_floor(target.routing(), paid_price, settled_amount, context) .await?; // ADR-0004 observe-only telemetry: log off-curve quotes only AFTER the @@ -1445,14 +1602,17 @@ impl PaymentVerifier { /// honest client may overpay a cheap quote to clear stricter receivers). async fn verify_legacy_median_candidate( &self, - xorname: &XorName, + target: &PaymentTarget, candidate: LegacyMedianCandidate<'_>, ) -> Result { - Self::validate_paid_quote_content(xorname, candidate)?; + // The two checks take different addresses. The quote must name what was + // paid for; the issuer must be close to what the network routes. A + // chunk supplies one address for both, a pointer two. + Self::validate_paid_quote_content(target.content(), candidate)?; let issuer_peer_id = Self::validate_paid_quote_peer_binding(candidate.encoded_peer_id, candidate.quote)?; - self.validate_paid_quote_issuer_k_closest(xorname, &issuer_peer_id) + self.validate_paid_quote_issuer_k_closest(target.routing(), &issuer_peer_id) .await?; Self::validate_paid_quote_signature(candidate).await?; @@ -3186,10 +3346,28 @@ impl PaymentVerifier { #[allow(clippy::too_many_lines)] async fn verify_merkle_payment( &self, - xorname: &XorName, + target: &PaymentTarget, proof_bytes: &[u8], context: VerificationContext, ) -> Result<()> { + // The proof names what was paid for, which for a pointer is its state + // rather than its address. + let xorname = target.content(); + + // A merkle proof binds the paid address but carries no issuer-proximity + // check, so it cannot express "paid at the state, quoted by the group + // around the address". Accepting one for a pointer would check the + // close group of a state identifier, which names no group at all. + // Single-node proofs do carry that check, so pointers use those until + // the merkle proof shape can say which group issued it. + if target.routing() != target.content() { + return Err(Error::Payment(format!( + "a pointer update must be paid with a single-node proof: a merkle \ + proof cannot bind the issuing close group of {} to the paid state {}", + hex::encode(target.routing()), + hex::encode(target.content()) + ))); + } if crate::logging::enabled!(crate::logging::Level::DEBUG) { debug!( "Verifying merkle payment for {} ({context:?})", @@ -3596,6 +3774,61 @@ mod tests { /// Create a verifier for unit tests. EVM is always on, but tests can /// pre-populate the cache to bypass on-chain verification. + /// A chunk whose address equals a pointer's `state_id` must not be able to + /// pay for that pointer's update. + /// + /// Reaching a pointer's `state_id` with a chunk now takes a BLAKE3 + /// preimage under one BLAKE3 mode for an output of the other — it is a + /// `derive_key` output, and a chunk address is a plain hash. This is the second line: were the two ever to + /// meet at one value, filing both "already paid" entries under it would let + /// chunk price buy a pointer update, skipping issuer proximity, the price + /// floor and the proof-shape rule with it. The typed key holds whatever the + /// two hash modes do. + #[test] + fn a_chunk_cannot_pay_for_a_pointer_that_shares_its_address() { + let state_id: XorName = [0x5Au8; 32]; + let pointer_address: XorName = [0xA5u8; 32]; + + let chunk = PaymentTarget::same(state_id); + let pointer = PaymentTarget::split(pointer_address, state_id); + + assert_eq!( + chunk, + PaymentTarget::Chunk(state_id), + "a chunk keeps its bare address, so existing entries are untouched" + ); + assert_ne!( + pointer, chunk, + "the pointer must not read the chunk's paid entry as its own" + ); + + // And two different pointers sharing a state cannot borrow either. + let other = PaymentTarget::split([0x11u8; 32], state_id); + assert_ne!(pointer, other); + + // The key is typed, not hashed: there is no 32-byte preimage a client + // could put in a chunk's *content* to land on the pointer's entry, + // because no chunk key is ever a `PointerState` variant. + let cache = VerifiedCache::with_capacity(8); + cache.insert_key(chunk); + assert!(cache.contains_key(&chunk), "the chunk's own entry is there"); + assert!( + !cache.contains_key(&pointer), + "and it does not satisfy the pointer" + ); + for crafted in [state_id, pointer_address, [0u8; 32], [0xFFu8; 32]] { + assert!( + !cache.contains_key(&PaymentTarget::split(pointer_address, state_id)), + "no chunk address {crafted:?} can stand in for the pointer's entry" + ); + cache.insert_key(PaymentTarget::Chunk(crafted)); + } + assert!( + !cache.contains_key(&pointer), + "still no chunk address satisfies the pointer's typed entry" + ); + } + fn create_test_verifier() -> PaymentVerifier { let config = PaymentVerifierConfig { evm: EvmVerifierConfig::default(), diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs new file mode 100644 index 00000000..c4787f39 --- /dev/null +++ b/src/pointer/mod.rs @@ -0,0 +1,60 @@ +//! Pointers — paid mutable references with an immutable owner. +//! +//! Implements `docs/adr/ADR-0016-pointers-immutable-owner.md`. +//! +//! A pointer is a mutable, owner-signed reference stored at an address derived +//! from the owner's public key. Ownership is fixed at creation: there is no +//! transfer, no lineage, no certificates and no key rotation. That choice is +//! what lets the design be this small — the owner key is inlined in the +//! record, so validating a pointer needs nothing but the pointer. +//! +//! # What lives here +//! +//! The record itself is a **wire type** and lives in +//! [`ant_protocol::pointer`]: its encoding, identifiers and merge rule are +//! things the client and the node must agree on byte for byte. This module is +//! the node's half. +//! +//! - [`store`] — durable storage with merge-on-put, which is the part the +//! immutable chunk store cannot do. +//! - [`service`] — the request handler: validate, check payment, merge. +//! +//! # The two identifiers +//! +//! | Name | Derivation | Job | +//! |---|---|---| +//! | `A` | `derive_key("autonomi.pointer.address.v1", owner)` | routes, and decides which nodes are responsible | +//! | `state_id` | `derive_key("autonomi.pointer.state.v1", body)` | names the state, and is what a quote is paid against | +//! +//! BLAKE3's derive-key mode, not a hash of a prefix and the input. A chunk is +//! addressed by a plain hash of its content, so a prefix would put both of +//! these inside the chunk address space for anyone who could write the +//! preimage — squatting an address before its owner used it, or buying a +//! pointer and a chunk with one payment. The two modes still share a 32-byte +//! range; what changed is that crossing it is a preimage problem. +//! +//! They are separate because `A` must be stable for the pointer's life while +//! the paid identifier must change with every update, or updates after the +//! first would be free — the 1.0 defect this design exists to fix. +//! +//! # Why the merge rule ignores signature bytes +//! +//! ML-DSA signing in `saorsa-pqc` is randomized and exposes no deterministic +//! mode, so one authenticated state has unboundedly many valid encodings. A +//! merge rule that ordered record *bytes* would let an owner sign one paid +//! state repeatedly, sort worst-first, and have every submission win — +//! unbounded storage, replication and Merkle-rebuild work for a single +//! payment. So equal state never replaces, and replicas may legitimately hold +//! different encodings of one state. Nothing compares record bytes across +//! replicas; `state_id` is compared instead. + +pub mod service; +pub mod store; + +pub use ant_protocol::pointer::{ + pointer_address, state_id_for_body, ParsedPointer, Pointer, PointerError, PointerState, + PointerTarget, PointerTargetKind, DATA_TYPE_POINTER, POINTER_BODY_LEN, POINTER_FORMAT_VERSION, + POINTER_WIRE_LEN, TARGET_WIRE_LEN, +}; +pub use service::PointerService; +pub use store::{PointerStore, PutOutcome}; diff --git a/src/pointer/service.rs b/src/pointer/service.rs new file mode 100644 index 00000000..bdda6c54 --- /dev/null +++ b/src/pointer/service.rs @@ -0,0 +1,817 @@ +//! The node's pointer request handler. +//! +//! Sits between the wire messages in [`ant_protocol::chunk`] and the +//! [`PointerStore`], and owns the three things a pointer PUT needs that a chunk +//! PUT does not: +//! +//! 1. **Payment at `state_id`, not at the address.** A pointer's address is +//! stable for its life, so quoting against it would make every update after +//! the first free — 1.0's defect. The quote's content is the state +//! identifier; the close group that answers is still the one around the +//! address. +//! 2. **Merge instead of "already exists".** The chunk path answers +//! `AlreadyExists` and stops, which for a mutable record silently drops +//! every update. +//! 3. **Cross-kind collision refusal.** A pointer address and a chunk address +//! are both 32 bytes from the same range; the domain separator makes a +//! collision infeasible, not impossible. A node that holds one kind at an +//! address refuses the other rather than silently choosing. +//! +//! # Order of work +//! +//! ```text +//! parse → compare with what is held → verify signature → check payment → commit +//! ``` +//! +//! The comparison precedes the signature check, so re-submitting a state the +//! node already holds costs a parse and a map lookup rather than an ML-DSA +//! verification. The payment check sits between validation and commit, and the +//! commit re-checks, because a newer state can land while payment is verified. + +use std::sync::Arc; + +use ant_protocol::chunk::{ + PointerGetRequest, PointerGetResponse, PointerPutRequest, PointerPutResponse, ProtocolError, + XorName, +}; +use bytes::Bytes; +use parking_lot::RwLock; +use saorsa_core::P2PNode; +use tokio::sync::mpsc; + +use crate::error::{Error, Result}; +use crate::logging::{debug, info, warn}; +use crate::payment::PaymentVerifier; +use crate::pointer::store::{Inspected, PointerStore, PutOutcome}; +use crate::replication::admission; +use crate::replication::pointer::PointerFreshWrite; +use crate::storage::{ChunkStore, SELF_CLOSENESS_GATE_WIDTH}; +use ant_protocol::pointer::POINTER_WIRE_LEN; + +/// Handles pointer requests against a [`PointerStore`]. +#[derive(Clone)] +pub struct PointerService { + /// Where pointers are kept. + store: PointerStore, + /// Consulted so an address held as a chunk is never overwritten by a + /// pointer. + chunks: Option>, + /// Confirms a state was paid for. `None` in tests that exercise the merge + /// rather than the payment. + payments: Option>, + /// Where a newly stored paid state goes to be offered to the rest of its + /// close group (ADR-0016 replication). Attached once the replication + /// engine exists, which is after this service is built; empty where + /// nothing replicates, as in unit tests and the devnet. + fresh_writes: Arc>>>, + /// The node's P2P handle, for the self-closeness gate. + /// + /// Attached after construction, because the node builds its protocol + /// handler before it has a running P2P node. `None` in unit tests that + /// never attach one, exactly as the chunk path does. + p2p_node: Arc>>>, +} + +impl std::fmt::Debug for PointerService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PointerService") + .field("records", &self.store.len()) + .field("checks_chunk_collisions", &self.chunks.is_some()) + .field("verifies_payment", &self.payments.is_some()) + .finish_non_exhaustive() + } +} + +impl PointerService { + /// Build a service over `store`. + #[must_use] + pub fn new(store: PointerStore) -> Self { + Self { + store, + chunks: None, + payments: None, + fresh_writes: Arc::new(RwLock::new(None)), + p2p_node: Arc::new(RwLock::new(None)), + } + } + + /// Apply the self-closeness gate, so a pointer PUT is admitted only where + /// the node is actually responsible. + /// + /// Without it one valid proof can be replayed to every node on the network, + /// each of which would store the record and trigger replication for an + /// address it has no business holding. + pub fn attach_p2p_node(&self, p2p_node: Arc) { + *self.p2p_node.write() = Some(p2p_node); + } + + /// Consult `chunks` before storing, so a cross-kind address collision is + /// refused rather than resolved by whichever kind arrived last. + #[must_use] + pub fn with_chunk_store(mut self, chunks: Arc) -> Self { + self.chunks = Some(chunks); + self + } + + /// Require payment, verified against each record's `state_id`. + #[must_use] + pub fn with_payments(mut self, payments: Arc) -> Self { + self.payments = Some(payments); + self + } + + /// Hand every newly stored paid state to replication on `writes`. + pub fn attach_fresh_writes(&self, writes: mpsc::UnboundedSender) { + *self.fresh_writes.write() = Some(writes); + } + + /// The store this service fronts. + #[must_use] + pub const fn store(&self) -> &PointerStore { + &self.store + } + + /// Handle a pointer PUT. + /// + /// Never returns `Err`: a rejection is a response the peer should see, so + /// every failure is mapped onto [`PointerPutResponse`]. + pub async fn handle_put(&self, request: PointerPutRequest) -> PointerPutResponse { + // Cheap first: parse and compare against what is held. No signature is + // checked yet, so a forged record for an address this node does not + // serve is rejected by the gates below without buying an ML-DSA + // verification. + let parsed = match self.store.inspect(&request.record).await { + Ok(Inspected::Unchanged(state)) => { + return PointerPutResponse::Unchanged { + address: state.address, + state_id: state.state_id, + }; + } + Ok(Inspected::Stale(state)) => { + // The state named is the one this node keeps, not the one that + // arrived: it is what the sender needs in order to catch up. + return PointerPutResponse::Stale { + address: state.address, + state_id: self.store.state_id(&state.address).unwrap_or_default(), + }; + } + Ok(Inspected::Candidate(parsed)) => parsed, + Err(e) => { + debug!("Pointer PUT refused: {e}"); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }; + + let address = parsed.state().address; + let state_id = parsed.state().state_id; + + if let Some(refusal) = self.admit(parsed.state()).await { + return refusal; + } + + // Only now is the record worth a signature check. + let record = match self.store.verify(parsed).await { + Ok(record) => record, + Err(e) => { + debug!("Pointer PUT refused: {e}"); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }; + + // Payment is checked against the state, not the address: the address + // never changes, so paying against it would buy every future update. + if let Some(payments) = &self.payments { + if let Err(e) = + Self::verify_payment(payments, address, state_id, request.payment_proof.as_ref()) + .await + { + return PointerPutResponse::PaymentRequired { + message: e.to_string(), + }; + } + } + + // Charge the bytes this write will take, and hold the charge until it + // lands. Checking capacity and then writing is the race the file store + // exists to close: concurrent writers all pass one cached measurement + // before any of them has written a byte, and cross the reserve + // together. A record is a fixed `POINTER_WIRE_LEN`, so that is the + // whole charge. + let reservation = match &self.chunks { + Some(chunks) => match chunks.reserve(POINTER_WIRE_LEN as u64) { + Ok(reservation) => Some(reservation), + Err(e) => { + debug!("Rejecting pointer PUT for {}: {e}", hex::encode(address)); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }, + None => None, + }; + // The charge travels with the write. Settling it here instead would + // release it the moment this future is dropped, while the blocking + // transaction it started runs on and publishes the file. + let record_bytes = request.record.to_vec(); + match self.store.commit(record, reservation).await { + Ok(PutOutcome::Changed) => { + // Offer it to the rest of the close group, proof included, so + // every member holds it whichever of them the client reached. + let writes = self.fresh_writes.read().clone(); + if let (Some(writes), Some(proof)) = (writes, request.payment_proof) { + if writes + .send(PointerFreshWrite { + record: record_bytes, + payment_proof: proof, + }) + .is_err() + { + debug!( + "Replication is not running; pointer {} is not offered on", + hex::encode(address) + ); + } + } + PointerPutResponse::Success { address, state_id } + } + // The re-check under the commit lock found a newer state. The + // client paid for a state that lost a race; say so plainly. + Ok(PutOutcome::Unchanged) => PointerPutResponse::Unchanged { address, state_id }, + Ok(PutOutcome::Stale) => PointerPutResponse::Stale { + address, + state_id: self.store.state_id(&address).unwrap_or(state_id), + }, + Err(e) => { + warn!("Pointer commit failed for {}: {e}", hex::encode(address)); + PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())) + } + } + } + + /// Everything that must hold before a record is worth verifying. + /// + /// `Some(response)` means refuse. Ordered cheapest first, and all of it + /// ahead of the signature check, so a forged record for an address this + /// node does not serve buys no cryptography. + async fn admit( + &self, + state: &ant_protocol::pointer::PointerState, + ) -> Option { + let address = state.address; + + // Any state the merge rule prefers to what this node knows, whatever + // its counter: a node that missed updates, or joined the group after + // them, must take the next one or it never catches up. An arrival that + // loses to what is held was already answered as stale; this also + // covers a record the node lost, which that comparison cannot see, so + // a replay cannot roll it back. + if !self.store.admits(state) { + info!( + "Rejecting pointer PUT for {}: counter {} does not beat the state this node knows", + hex::encode(address), + state.counter + ); + return Some(PointerPutResponse::Stale { + address, + state_id: self.store.state_id(&address).unwrap_or_default(), + }); + } + + if let Some(chunks) = &self.chunks { + // A chunk already here means the two kinds collided. Refuse rather + // than pick: whichever we chose, someone's data would vanish. + if let Some(refusal) = cross_kind_refusal(address, chunks.exists(&address)) { + return Some(refusal); + } + // Capacity before payment, as the chunk path does, and for the + // size a record actually is rather than for nothing. The binding + // charge is taken at the commit; this only avoids paying to find + // out the disk is full. + if let Err(e) = chunks.check_capacity_for(POINTER_WIRE_LEN as u64) { + let addr = hex::encode(address); + info!( + target: "ant_node::storage::disk_precheck", + addr = %addr, + kind = "pointer", + "Rejecting pointer PUT before payment verification: {e}" + ); + return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + e.to_string(), + ))); + } + } + + // Self-closeness gate (ADR-0003), judged at the pointer's address + // because that is what the network routes on. Bind the handle out of + // the lock first: no guard may be held across an await. + let attached = self.p2p_node.read().as_ref().map(Arc::clone); + if let Some(p2p) = attached { + let self_id = *p2p.peer_id(); + if !admission::is_responsible(&self_id, &address, &p2p, SELF_CLOSENESS_GATE_WIDTH).await + { + debug!( + "Rejecting pointer PUT for {}: not within local closest peers", + hex::encode(address) + ); + return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + "node is not within its local closest peers for this address".to_string(), + ))); + } + } + None + } + + /// Handle a pointer GET. + pub async fn handle_get(&self, request: PointerGetRequest) -> PointerGetResponse { + match self.store.get(&request.address).await { + Ok(Some(record)) => PointerGetResponse::Success { + record: Bytes::from(record.to_bytes()), + }, + Ok(None) => PointerGetResponse::NotFound { + address: request.address, + }, + Err(e) => PointerGetResponse::Error(ProtocolError::StorageFailed(e.to_string())), + } + } + + /// Confirm the submitted state was paid for. + /// + /// `routing_address` selects the close group whose quotes count; + /// `paid_content` is what the quote must name. They are different values + /// here, which is the whole point — the existing chunk path passes one + /// address for both jobs. + async fn verify_payment( + payments: &PaymentVerifier, + routing_address: XorName, + paid_content: XorName, + proof: Option<&Vec>, + ) -> Result<()> { + let Some(proof) = proof else { + return Err(Error::Payment(format!( + "a pointer update must be paid for; no proof supplied for state {}", + hex::encode(paid_content) + ))); + }; + payments + .verify_pointer_payment(&routing_address, &paid_content, proof) + .await + } +} + +/// Decide whether a chunk already at `address` blocks this pointer. +/// +/// Separated from the handler because the branch cannot be reached in a test +/// any other way: a pointer address comes out of BLAKE3's derive-key mode and a +/// chunk address out of a plain hash, so occupying both with real data would +/// take a collision across the two. The decision is what matters, so the +/// decision is what is tested. +/// +/// `Some(response)` means refuse. Refusing is the only safe answer: whichever +/// kind were chosen, the other's data would be destroyed, and the node cannot +/// know which one the network expects. +fn cross_kind_refusal(address: XorName, chunk_present: Result) -> Option { + match chunk_present { + Ok(true) => { + warn!( + "Refusing pointer at {}: a chunk already occupies that address", + hex::encode(address) + ); + Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + format!( + "address {} is already occupied by a chunk; refusing to store a \ + pointer over it", + hex::encode(address) + ), + ))) + } + Ok(false) => None, + Err(e) => Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + format!("cannot check the chunk store for a collision: {e}"), + ))), + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + reason = "test assertions" +)] +mod tests { + use super::*; + use crate::payment::{EvmVerifierConfig, PriceFloorConfig}; + use ant_protocol::pointer::{Pointer, PointerTarget, PointerTargetKind}; + use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; + + fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) + } + + fn signed(seed: u8, counter: u64, target_byte: u8) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(PointerTargetKind::Chunk, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + async fn service() -> (PointerService, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + (PointerService::new(store), dir) + } + + /// A service whose chunk store guards a disk that cannot take another byte. + async fn service_with_full_disk() -> (PointerService, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + // Larger than any disk, so every capacity question answers "full". + disk_reserve: u64::MAX, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + ( + PointerService::new(store).with_chunk_store(Arc::new(chunks)), + dir, + ) + } + + #[tokio::test] + async fn a_full_disk_refuses_a_pointer_before_it_is_written() { + // A disk with no room refuses at the admission check, before payment, + // so a client is not charged to find out. This does not reach the + // reservation — the store-level tests cover that — it covers the early + // refusal and that nothing is stored when it fires. + let (service, _dir) = service_with_full_disk().await; + let record = signed(1, 0, 1); + + match service.handle_put(put(&record)).await { + PointerPutResponse::Error(ProtocolError::StorageFailed(message)) => { + assert!( + message.contains("disk space") || message.contains("reserve"), + "a full disk should say so, got: {message}" + ); + } + other => panic!("a full disk must refuse the write, got {other:?}"), + } + + assert!( + matches!( + service + .handle_get(PointerGetRequest::new(record.address())) + .await, + PointerGetResponse::NotFound { .. } + ), + "nothing may be stored when the disk had no room for it" + ); + } + + #[tokio::test] + async fn concurrent_creations_are_each_charged_against_the_disk() { + // Every one of these would pass a bare capacity *check* against one + // cached measurement. What stops them collectively crossing the + // reserve is that each holds a charge until its write lands. + let (service, dir) = service().await; + let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + let service = service.with_chunk_store(Arc::new(chunks)); + + let chunks = service.chunks.clone().expect("chunk store"); + let (written_before, in_flight_before) = chunks.capacity_counters(); + + let mut writes = Vec::new(); + for seed in 1..=8u8 { + let record = signed(seed, 0, 1); + let service = service.clone(); + writes.push(tokio::spawn(async move { + (record.address(), service.handle_put(put(&record)).await) + })); + } + + for write in writes { + let (address, response) = write.await.expect("join"); + assert!( + matches!(response, PointerPutResponse::Success { .. }), + "a disk with room must take the write, got {response:?}" + ); + assert!(matches!( + service.handle_get(PointerGetRequest::new(address)).await, + PointerGetResponse::Success { .. } + )); + } + assert_eq!(service.store().len(), 8); + + // Each of the eight was charged, and none of the charges was left + // hanging. Without the counters this would pass just as well against a + // bare capacity check that charges nothing. + // Exactly eight allocation charges. An inequality against the raw + // record size would accept four: a record rounds up to the allocation + // unit, and four of those already exceed eight record sizes. + let one = ChunkStore::capacity_charge_for(POINTER_WIRE_LEN as u64); + assert_eq!( + chunks.capacity_counters(), + (written_before + 8 * one, in_flight_before), + "eight writes, eight charges, none left in flight" + ); + } + + fn put(record: &Pointer) -> PointerPutRequest { + PointerPutRequest::new(Bytes::from(record.to_bytes())) + } + + #[tokio::test] + async fn a_put_then_a_get_round_trips_the_record() { + let (service, _dir) = service().await; + let record = signed(1, 0, 1); + + match service.handle_put(put(&record)).await { + PointerPutResponse::Success { address, state_id } => { + assert_eq!(address, record.address()); + assert_eq!(state_id, record.state_id()); + } + other => panic!("expected Success, got {other:?}"), + } + + match service + .handle_get(PointerGetRequest::new(record.address())) + .await + { + PointerGetResponse::Success { record: bytes } => { + assert_eq!(bytes.as_ref(), record.to_bytes()); + } + other => panic!("expected Success, got {other:?}"), + } + } + + #[tokio::test] + async fn an_update_replaces_and_a_stale_one_is_told_so() { + let (service, _dir) = service().await; + let first = signed(1, 0, 1); + let second = signed(1, 1, 1); + service.handle_put(put(&first)).await; + + assert!(matches!( + service.handle_put(put(&second)).await, + PointerPutResponse::Success { .. } + )); + + match service.handle_put(put(&first)).await { + PointerPutResponse::Stale { address, state_id } => { + assert_eq!(address, first.address()); + assert_eq!(state_id, second.state_id(), "the newer state is reported"); + } + other => panic!("expected Stale, got {other:?}"), + } + } + + #[tokio::test] + async fn two_nodes_given_the_same_states_in_opposite_orders_agree() { + // The convergence property, driven through the request handler rather + // than the store. The gate in front of the merge rule is part of the + // production path, and a gate that admits records by arrival order + // would leave these two nodes holding different records for ever -- + // which is the fork the merge rule exists to prevent. + let (first_node, _a) = service().await; + let (second_node, _b) = service().await; + + // Two separately paid states at one counter. The merge rule says the + // smaller target wins, whichever arrives first. + let loser = signed(1, 0, 9); + let winner = signed(1, 0, 1); + assert!(winner.replaces(&loser)); + + for record in [&loser, &winner] { + first_node.handle_put(put(record)).await; + } + for record in [&winner, &loser] { + second_node.handle_put(put(record)).await; + } + + for (name, node) in [("first", &first_node), ("second", &second_node)] { + match node + .handle_get(PointerGetRequest::new(winner.address())) + .await + { + PointerGetResponse::Success { record: bytes } => assert_eq!( + Pointer::from_bytes(&bytes).expect("parse").state_id(), + winner.state_id(), + "the {name} node kept the wrong record" + ), + other => panic!("expected the winner, got {other:?}"), + } + } + + // And the node that already had the winner refused to go back. + assert!(matches!( + second_node.handle_put(put(&loser)).await, + PointerPutResponse::Stale { .. } + )); + } + + #[tokio::test] + async fn a_fork_between_two_nodes_is_healed_by_any_later_counter() { + // Two paid states at one counter, each reaching a different node: a + // fork no read can settle for them. The next update, at any later + // counter, lands on both and leaves them holding the same record. + let (first_node, _a) = service().await; + let (second_node, _b) = service().await; + let one_side = signed(1, 1, 9); + let other_side = signed(1, 1, 1); + assert!(matches!( + first_node.handle_put(put(&one_side)).await, + PointerPutResponse::Success { .. } + )); + assert!(matches!( + second_node.handle_put(put(&other_side)).await, + PointerPutResponse::Success { .. } + )); + + let healed = signed(1, 5, 4); + for (name, node) in [("first", &first_node), ("second", &second_node)] { + match node.handle_put(put(&healed)).await { + PointerPutResponse::Success { address, state_id } => { + assert_eq!(address, healed.address()); + assert_eq!(state_id, healed.state_id()); + } + other => panic!("the {name} node refused the healing update: {other:?}"), + } + let held = node + .store() + .get(&healed.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.state_id(), healed.state_id(), "the {name} node"); + } + } + + #[tokio::test] + async fn re_submitting_a_held_state_is_unchanged_not_success() { + // A client that retries must be able to tell "your update landed" from + // "you paid for something already stored". + let (service, _dir) = service().await; + let record = signed(1, 0, 4); + service.handle_put(put(&record)).await; + + let variant = signed(1, 0, 4); + assert_ne!(variant.to_bytes(), record.to_bytes()); + match service.handle_put(put(&variant)).await { + PointerPutResponse::Unchanged { address, state_id } => { + assert_eq!(address, record.address()); + assert_eq!(state_id, record.state_id()); + } + other => panic!("expected Unchanged, got {other:?}"), + } + } + + #[tokio::test] + async fn an_absent_pointer_is_not_found() { + let (service, _dir) = service().await; + match service.handle_get(PointerGetRequest::new([7u8; 32])).await { + PointerGetResponse::NotFound { address } => assert_eq!(address, [7u8; 32]), + other => panic!("expected NotFound, got {other:?}"), + } + } + + #[tokio::test] + async fn junk_is_refused_rather_than_stored() { + let (service, _dir) = service().await; + assert!(matches!( + service + .handle_put(PointerPutRequest::new(Bytes::from_static(b"not a pointer"))) + .await, + PointerPutResponse::Error(_) + )); + assert!(service.store().is_empty()); + } + + #[tokio::test] + async fn a_forged_signature_is_refused() { + let (service, _dir) = service().await; + let record = signed(1, 0, 1); + let mut bytes = record.to_bytes(); + if let Some(byte) = bytes.get_mut(ant_protocol::pointer::POINTER_BODY_LEN + 3) { + *byte ^= 0xff; + } + assert!(matches!( + service + .handle_put(PointerPutRequest::new(Bytes::from(bytes))) + .await, + PointerPutResponse::Error(_) + )); + assert!(service.store().is_empty()); + } + + #[tokio::test] + async fn payment_is_required_when_a_verifier_is_attached() { + use crate::payment::{PaymentVerifier, PaymentVerifierConfig}; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + let verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { + evm: EvmVerifierConfig::default(), + cache_capacity: 16, + close_group_size: ant_protocol::chunk::CLOSE_GROUP_SIZE, + local_rewards_address: evmlib::common::Address::new([1u8; 20]), + price_floor: PriceFloorConfig::default(), + })); + let service = PointerService::new(store).with_payments(verifier); + + // No proof at all: refused as PaymentRequired, not stored. + match service.handle_put(put(&signed(1, 0, 1))).await { + PointerPutResponse::PaymentRequired { message } => { + assert!(message.contains("must be paid for"), "got: {message}"); + } + other => panic!("expected PaymentRequired, got {other:?}"), + } + assert!( + service.store().is_empty(), + "an unpaid update must not be stored" + ); + } + + #[tokio::test] + async fn any_counter_that_beats_what_is_held_is_taken() { + let (service, _dir) = service().await; + + // A node that joined after the pointer was created holds nothing, so + // the first record it sees need not be counter 0. + let first_seen = signed(1, 5, 1); + assert!(matches!( + service.handle_put(put(&first_seen)).await, + PointerPutResponse::Success { .. } + )); + + // Anything older is stale, whatever else it says. + for older in [0u64, 4] { + assert!( + matches!( + service.handle_put(put(&signed(1, older, 2))).await, + PointerPutResponse::Stale { .. } + ), + "counter {older} must not replace counter 5" + ); + } + + // Anything newer lands, however far it skips. + for newer in [6u64, 99, u64::MAX] { + let record = signed(1, newer, 2); + match service.handle_put(put(&record)).await { + PointerPutResponse::Success { state_id, .. } => { + assert_eq!(state_id, record.state_id()); + } + other => panic!("counter {newer} was refused: {other:?}"), + } + } + let held = service + .store() + .get(&first_seen.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), u64::MAX); + } + + #[test] + fn a_pointer_is_refused_where_a_chunk_already_sits() { + // The two kinds share one 32-byte address space. A collision is + // infeasible, not impossible, and silently picking one would destroy + // the other's data. + let address = [3u8; 32]; + match cross_kind_refusal(address, Ok(true)) { + Some(PointerPutResponse::Error(ProtocolError::StorageFailed(message))) => { + assert!( + message.contains("already occupied by a chunk"), + "got: {message}" + ); + } + other => panic!("a collision must be refused, got {other:?}"), + } + } + + #[test] + fn a_free_address_is_not_refused() { + assert!(cross_kind_refusal([3u8; 32], Ok(false)).is_none()); + } + + #[test] + fn an_unreadable_chunk_store_refuses_rather_than_assumes() { + // "I could not check" must not be treated as "nothing is there". + let refusal = cross_kind_refusal([3u8; 32], Err(Error::Storage("disk gone".into()))); + match refusal { + Some(PointerPutResponse::Error(ProtocolError::StorageFailed(message))) => { + assert!(message.contains("cannot check"), "got: {message}"); + } + other => panic!("expected a refusal, got {other:?}"), + } + } +} diff --git a/src/pointer/store.rs b/src/pointer/store.rs new file mode 100644 index 00000000..3628d173 --- /dev/null +++ b/src/pointer/store.rs @@ -0,0 +1,2231 @@ +//! Durable storage for pointer records, with merge-on-put. +//! +//! The chunk store answers "already have it" and stops, which is right for an +//! immutable record and wrong for a mutable one — it would silently drop every +//! update. It also requires `BLAKE3(content) == address`, which no pointer can +//! satisfy. So pointers get their own store rather than an exemption carved +//! into that one. +//! +//! # The shape of a write +//! +//! A put is three steps, because the caller's gates sit between them: +//! +//! 1. `inspect` parses and compares against what is held: the arrival is +//! either unchanged, stale, or a candidate that would win. +//! 2. `verify` checks the candidate's signature — after the caller's +//! admission gates, and before it verifies payment. +//! 3. `commit` writes it, and settles the disk charge taken for it. +//! +//! Those three are internal to the crate; [`PointerStore::put_bytes`] is the +//! same sequence in one call, for callers with nothing to do in between. +//! +//! Nothing cheap happens after something expensive: a resubmission of what is +//! held is refused before any signature check, so repeatedly submitting one +//! paid state buys no ML-DSA verifications. Step 2 runs **outside** any store +//! lock, so a flood of unpaid candidates cannot block every other address +//! behind one verification, and step 3 re-checks under the lock, because the +//! world may have moved while payment was being verified. +//! +//! # Atomicity +//! +//! The commit — compare, write, re-index — runs inside a single blocking task +//! holding a synchronous lock. Splitting it across an `await` would let a +//! cancelled caller release the lock while its write was still in flight, +//! leaving the index describing a record the disk no longer holds. Because the +//! whole transaction lives in one task, dropping the caller's future cannot +//! tear it. +//! +//! On disk each record is one file named by its hex address, under +//! `{root}/pointers//`, where the shard is the address's last byte in +//! hex: 256 directories, as the chunk store keeps, so no directory grows with +//! the whole store. The last byte rather than the first because a node holds +//! addresses near its own identity, and their leading bytes cluster. Writes go +//! to a uniquely named temporary file in the shard and are renamed into place, +//! so a crash leaves either the old record or the new one, never a torn one. A +//! lock file under `{root}/pointers/` keeps two processes from keeping two +//! indexes over one set of files. + +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use fs2::FileExt; +use parking_lot::Mutex; +use tokio::task::spawn_blocking; + +use crate::ant_protocol::XorName; +use crate::error::{Error, Result}; +use crate::logging::{debug, warn}; +use ant_protocol::pointer::{ParsedPointer, Pointer, PointerState, POINTER_WIRE_LEN}; + +use crate::storage::{rename_with_retry, Reservation}; + +/// Directory under the store root that holds pointer records. +const POINTERS_DIR_NAME: &str = "pointers"; + +/// Prefix for the temporary file a write lands in before being renamed. +const TEMP_PREFIX: &str = ".tmp-"; + +/// Name of the file whose lock grants exclusive use of the directory. +const LOCK_FILE_NAME: &str = ".pointer-store-lock"; + +/// How many shard directories records are spread across: one per value of an +/// address's last byte. +const SHARD_COUNT: u16 = 256; + +/// How long a replaced record is kept after an update, in memory. +/// +/// A storage audit binds the record a node holds in its first round and asks +/// for it in the second (ADR-0016). An owner updating the pointer between the +/// two would otherwise fail the honest node that took the update, so the +/// record the update replaced stays servable for longer than an audit session +/// lives. +pub const SUPERSEDED_RETENTION: Duration = Duration::from_mins(5); + +/// Most replaced records kept at once, about 11 MB at the cap. Past it the +/// oldest goes first; reaching it inside [`SUPERSEDED_RETENTION`] takes that +/// many paid updates to pointers this node holds. +const MAX_SUPERSEDED: usize = 2048; + +/// The name of the shard directory `address` lives in: its last byte in hex. +fn shard_name(address: &XorName) -> String { + let last = address.last().copied().unwrap_or_default(); + format!("{last:02x}") +} + +/// The shard directory `address` lives in. +fn shard_dir(dir: &Path, address: &XorName) -> PathBuf { + dir.join(shard_name(address)) +} + +/// The file that holds the record at `address`. +fn record_path(dir: &Path, address: &XorName) -> PathBuf { + shard_dir(dir, address).join(hex::encode(address)) +} + +/// What a put did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutOutcome { + /// The record was written: either nothing was held, or it won the merge. + /// + /// One outcome rather than two, because nothing downstream treats "stored" + /// differently from "replaced": both mean the node now holds this state. + Changed, + /// The held record is the same authenticated state. Nothing was written. + /// + /// This is the case that keeps one payment from funding many writes: an + /// owner can produce unboundedly many valid signatures over one state, and + /// every one of them lands here. + Unchanged, + /// The incoming record lost under the merge rule. Nothing was written. + Stale, +} + +/// What the store has done since it was opened, for telemetry. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PointerStoreStats { + /// Records written, whether new or replacing an older state. + pub written: u64, + /// Arrivals that carried the state already held. Nothing was written. + pub unchanged: u64, + /// Arrivals that lost to the state already held. Nothing was written. + pub stale: u64, + /// Records served to a reader. + pub served: u64, + /// Records removed because this node stopped being responsible for them. + pub deleted: u64, +} + +/// The live counters behind [`PointerStoreStats`]. +#[derive(Debug, Default)] +struct Counters { + written: AtomicU64, + unchanged: AtomicU64, + stale: AtomicU64, + served: AtomicU64, + deleted: AtomicU64, +} + +impl Counters { + fn count(&self, outcome: PutOutcome) { + let counter = match outcome { + PutOutcome::Changed => &self.written, + PutOutcome::Unchanged => &self.unchanged, + PutOutcome::Stale => &self.stale, + }; + counter.fetch_add(1, Ordering::Relaxed); + } + + fn snapshot(&self) -> PointerStoreStats { + PointerStoreStats { + written: self.written.load(Ordering::Relaxed), + unchanged: self.unchanged.load(Ordering::Relaxed), + stale: self.stale.load(Ordering::Relaxed), + served: self.served.load(Ordering::Relaxed), + deleted: self.deleted.load(Ordering::Relaxed), + } + } +} + +/// The result of [`PointerStore::inspect`]: what an arrival claims, before any +/// signature has been checked. +#[derive(Debug)] +pub(crate) enum Inspected { + /// The node already holds exactly this state. No signature check is owed: + /// a resubmission of what was paid for and a forgery of it are the same + /// no-op. + Unchanged(PointerState), + /// The arrival loses to what is held, so it changes nothing either. + Stale(PointerState), + /// The arrival would win as it stands. Its signature has **not** been + /// checked yet — pass it to [`PointerStore::verify`] once admission gates + /// have had their say. + Candidate(ParsedPointer), +} + +/// What the store knows about a held record without reading it back. +#[derive(Debug, Clone, Copy)] +struct IndexEntry { + /// The held record's authenticated state. + /// + /// The state itself rather than fields copied out of it, so the merge order + /// the store applies is the protocol's own rule applied to the held state, + /// and cannot drift from it. + state: PointerState, + /// Whether the file behind `state` is still there and still that record. + /// + /// A read that finds it gone or different clears this rather than dropping + /// the entry. The node stops serving the record, because it does not have + /// it — but it still knows what it had, and that is what lets the state be + /// restored. Dropping the entry would leave the address looking untouched, + /// where any record is admissible, so a replay of an older state could roll + /// the node back. + on_disk: bool, + /// Which insertion this entry is. + /// + /// Monotonic for the life of this store, so an entry can be told apart + /// from a later one carrying the same `state_id`. Without it a repair that + /// restores exactly the state a reader found corrupt would be + /// indistinguishable from the corrupt entry that reader set out to disown, + /// and a second reader would erase the repair. Reuse would take 2^64 + /// commits without a restart, and an observation never outlives the store + /// that produced it, so a restart resetting the counter is harmless. + generation: u64, +} + +impl IndexEntry { + /// Describe a validated record. + fn of(record: &Pointer, generation: u64) -> Self { + Self::held(record.state(), generation) + } + + /// Describe a record held on disk in `state`. + const fn held(state: PointerState, generation: u64) -> Self { + Self { + state, + on_disk: true, + generation, + } + } +} + +/// A store of pointer records. +#[derive(Debug, Clone)] +pub struct PointerStore { + inner: Arc, +} + +/// Shared state behind [`PointerStore`]. +#[derive(Debug)] +struct Inner { + /// Directory holding one file per record. + dir: PathBuf, + /// What is held, by address. + /// + /// Guarded by a synchronous lock held across the disk write, so the index + /// and the directory can never disagree about which version is current. + /// The critical section is one ~5 KB write, and it is only ever taken + /// inside a blocking task. + index: Mutex>, + /// Distinguishes concurrent temporary files so two writers to one address + /// cannot land in the same partial file. + write_seq: AtomicU64, + /// Source of index generations, monotonic for this store's lifetime. + generation: AtomicU64, + /// What the store has done, for telemetry. + counters: Counters, + /// The record each recent update replaced, by address, with when (see + /// [`SUPERSEDED_RETENTION`]). + superseded: Mutex)>>, + /// Held for the store's lifetime; releasing it releases the directory. + _lock_file: File, +} + +impl PointerStore { + /// Open a store under `root_dir`, rebuilding its index from disk. + /// + /// Takes an exclusive lock on the directory. Two stores over one directory + /// would each keep their own index, and the one with the staler view would + /// happily overwrite the other's newer record. + /// + /// A file that does not parse as a valid pointer, or that is not at the + /// address its owner derives, is skipped and logged rather than deleted: a + /// store that prunes what it cannot read destroys the evidence of its own + /// bug. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the directory cannot be created, locked or + /// read. + pub async fn new(root_dir: &Path) -> Result { + let dir = root_dir.join(POINTERS_DIR_NAME); + let scan_dir = dir.clone(); + let (lock_file, index) = spawn_blocking(move || { + std::fs::create_dir_all(&scan_dir).map_err(|e| { + Error::Storage(format!("cannot create {}: {e}", scan_dir.display())) + })?; + let lock_file = acquire_lock(&scan_dir)?; + create_shards(&scan_dir)?; + let index = scan(&scan_dir)?; + Ok::<_, Error>((lock_file, index)) + }) + .await + .map_err(|e| Error::Storage(format!("pointer store scan panicked: {e}")))??; + + let next_generation = index + .values() + .map(|entry| entry.generation) + .max() + .map_or(0, |highest| highest.saturating_add(1)); + debug!( + "Pointer store opened at {} with {} records", + dir.display(), + index.len() + ); + Ok(Self { + inner: Arc::new(Inner { + dir, + index: Mutex::new(index), + write_seq: AtomicU64::new(0), + generation: AtomicU64::new(next_generation), + counters: Counters::default(), + superseded: Mutex::new(HashMap::new()), + _lock_file: lock_file, + }), + }) + } + + /// Parse an arrival and decide whether it could change anything, without + /// verifying its signature. + /// + /// Cheap checks first: an arrival that cannot change anything is refused + /// before its signature is looked at, because it is a no-op whether or not + /// it is correctly signed. That ordering is what stops repeated submission + /// of one paid state from buying ML-DSA verifications. Callers that gate on + /// admission — capacity, responsibility for the address — run this first, + /// apply their + /// gates, and only then pay for [`Self::verify`]. Otherwise a forged record + /// for an address the node is not responsible for still buys an ML-DSA + /// verification before anything rejects it. + /// + /// # Errors + /// + /// Returns [`Error::Protocol`] if the bytes are not a well-formed record. + pub(crate) async fn inspect(&self, bytes: &[u8]) -> Result { + // Off the executor: deciding this reads the held record back off the + // disk, and a flood of arrivals must not put a blocking read on a + // runtime worker for each one. + let store = self.clone(); + let bytes = bytes.to_vec(); + spawn_blocking(move || store.inspect_blocking(&bytes)) + .await + .map_err(|e| Error::Storage(format!("pointer inspection panicked: {e}")))? + } + + /// The body of [`Self::inspect`], on a blocking thread. + fn inspect_blocking(&self, bytes: &[u8]) -> Result { + // Parsing decodes the owner key once; `verify` reuses that parse rather + // than decoding again. The bytes travel with it, so the two cannot be + // mismatched. + let parsed = ParsedPointer::parse(bytes.to_vec())?; + let state = *parsed.state(); + + // Both early answers below assert that this node holds something at + // least as good as what arrived, so neither may be given on the index's + // word alone. + if let Some(held) = self.held_state(&state.address) { + if held.state_id == state.state_id { + self.inner.counters.count(PutOutcome::Unchanged); + return Ok(Inspected::Unchanged(state)); + } + if !state.replaces(&held) { + self.inner.counters.count(PutOutcome::Stale); + return Ok(Inspected::Stale(state)); + } + } + Ok(Inspected::Candidate(parsed)) + } + + /// Verify a candidate's signature. + /// + /// Runs with no lock held and off the async executor: an ML-DSA + /// verification is milliseconds of CPU, and an attacker can ask for one per + /// forged record, so it must not run on a thread other work needs. + /// + /// # Errors + /// + /// Returns [`Error::Crypto`] if the signature does not verify. + pub(crate) async fn verify(&self, parsed: ParsedPointer) -> Result { + spawn_blocking(move || Pointer::verify_parsed(parsed)) + .await + .map_err(|e| Error::Storage(format!("pointer verification panicked: {e}")))? + .map_err(Into::into) + } + + /// Commit a prepared record. + /// + /// Re-checks against what is held before writing: `prepare` may have run + /// before a payment check that took long enough for a newer state to + /// arrive, and the re-check is what keeps that newer state from being + /// overwritten. + /// + /// `reservation` is the disk charge for this write, taken before the call. + /// It moves into the blocking transaction rather than staying with the + /// caller, because the caller's future can be dropped while that + /// transaction runs on: a charge released here would leave the file that + /// landed a moment later uncounted. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the write fails. + pub(crate) async fn commit( + &self, + record: Pointer, + reservation: Option, + ) -> Result { + let inner = Arc::clone(&self.inner); + // The whole transaction runs in one task, so dropping this future + // cannot leave the write done and the index un-updated. + let outcome = spawn_blocking(move || inner.commit_blocking(&record, reservation)) + .await + .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))??; + self.inner.counters.count(outcome); + Ok(outcome) + } + + /// Validate and store in one step, with no payment gate. + /// + /// For callers that have already settled payment, and for tests. The + /// request path runs the same three steps with its admission and payment + /// gates between them. + /// + /// # Errors + /// + /// As the three steps it runs: a malformed record, a signature that does + /// not verify, or a write that fails. + pub async fn put_bytes(&self, bytes: &[u8]) -> Result { + match self.inspect(bytes).await? { + Inspected::Unchanged(_) => Ok(PutOutcome::Unchanged), + Inspected::Stale(_) => Ok(PutOutcome::Stale), + Inspected::Candidate(parsed) => { + let record = self.verify(parsed).await?; + self.commit(record, None).await + } + } + } + + /// Read the record held at `address`, if any. + /// + /// Re-validates on the way out: the signature is checked and the record + /// must belong at the address asked for, so a file corrupted under the node + /// is reported as missing rather than served as authentic, and the entry is + /// disowned so the node will take a fresh copy instead of answering + /// "unchanged" to its own repair. + /// + /// The file is what is served, not the index. A file replaced under the + /// node by a *different* record that is genuinely signed for this address + /// is served as held — it is a real record, the reader verifies it, and the + /// read quorum is what decides between replicas that disagree. What the + /// index is not allowed to do is claim a state the file does not have — + /// that check guards `inspect`'s two early answers, the ones that + /// assert this node already holds something. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file cannot be read for a reason other + /// than its absence. + pub async fn get(&self, address: &XorName) -> Result> { + // What the index claimed before the read. A commit can land while the + // read is in flight, and its entry must not then be mistaken for the + // stale one this read is about to disown. + let claimed = self.snapshot(address).map(|entry| entry.generation); + + let path = self.path_for(address); + // Reading and verifying happen in one blocking task: an ML-DSA check is + // milliseconds of CPU and must not run on an async worker thread. + let read = spawn_blocking(move || { + read_record_file(&path).map(|bytes| bytes.map(|bytes| Pointer::from_bytes(&bytes))) + }) + .await + .map_err(|e| Error::Storage(format!("pointer read panicked: {e}")))?; + + let validated = match read { + Ok(Some(validated)) => validated, + Ok(None) => { + self.disown_if_unchanged(address, claimed); + return Ok(None); + } + Err(e) => return Err(e), + }; + + match validated { + Ok(record) if record.address() == *address => { + self.inner.counters.served.fetch_add(1, Ordering::Relaxed); + Ok(Some(record)) + } + Ok(_) => { + warn!( + "Pointer file at {} holds a record for another address; dropping it \ + from the index", + hex::encode(address) + ); + self.disown_if_unchanged(address, claimed); + Ok(None) + } + Err(e) => { + warn!( + "Pointer file at {} does not validate ({e}); dropping it from the index", + hex::encode(address) + ); + self.disown_if_unchanged(address, claimed); + Ok(None) + } + } + } + + /// The authenticated-state identifier held at `address`, if any. + /// + /// This is what a sync hint carries and what a fetch decision compares. It + /// names the state, not the encoding, so two replicas holding one state + /// under different signatures agree and do not refetch each other forever. + #[must_use] + pub fn state_id(&self, address: &XorName) -> Option { + self.snapshot(address) + .filter(|entry| entry.on_disk) + .map(|entry| entry.state.state_id) + } + + /// The record an update at `address` replaced within the last + /// [`SUPERSEDED_RETENTION`], if any: what a storage audit that bound it + /// before the update is still owed. + #[must_use] + pub fn superseded(&self, address: &XorName) -> Option> { + self.inner + .superseded + .lock() + .get(address) + .filter(|(at, _)| at.elapsed() < SUPERSEDED_RETENTION) + .map(|(_, bytes)| bytes.clone()) + } + + /// The bytes of the record held at `address`, read from disk without + /// verifying them. + /// + /// For a caller that only binds the bytes and has them verified later, by + /// whoever they are served to: verifying here would cost a signature check + /// per record for nothing. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists but cannot be read. + pub async fn record_bytes(&self, address: &XorName) -> Result>> { + let path = self.path_for(address); + spawn_blocking(move || read_record_file(&path)) + .await + .map_err(|e| Error::Storage(format!("pointer read panicked: {e}")))? + } + + /// The state held at `address`, as the index records it, if this node can + /// serve one. + /// + /// What a peer asking "which state do you hold?" is told. Taken from the + /// index rather than the file, so answering costs no disk read; a file lost + /// since is caught by the first read, which stops the index claiming it. + #[must_use] + pub fn state(&self, address: &XorName) -> Option { + self.snapshot(address) + .filter(|entry| entry.on_disk) + .map(|entry| entry.state) + } + + /// Whether a paid PUT of `state` may be taken. + /// + /// Any state the merge rule prefers to what is held, whatever its counter. + /// The counter orders states; it does not meter them, since each state is + /// paid for on its own. That is also what lets a node catch up: a write + /// ends once its quorum has answered, so a node can miss an update, and one + /// that joins the group later holds nothing at all. Either takes the next + /// update, however far ahead of it that is. + /// + /// A record this node knew and lost admits the lost state itself as well, + /// so the arrival that restores it is not mistaken for a resubmission. It + /// admits nothing older, so a node that lost its copy cannot be rolled + /// back by a replay. + #[must_use] + pub fn admits(&self, state: &PointerState) -> bool { + self.snapshot(&state.address).is_none_or(|entry| { + state.replaces(&entry.state) + || (!entry.on_disk && state.state_id == entry.state.state_id) + }) + } + + /// Whether a record is held at `address`. + #[must_use] + pub fn contains(&self, address: &XorName) -> bool { + self.snapshot(address).is_some_and(|entry| entry.on_disk) + } + + /// How many records the store holds. + #[must_use] + pub fn len(&self) -> usize { + self.inner + .index + .lock() + .values() + .filter(|entry| entry.on_disk) + .count() + } + + /// Whether the store holds nothing. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Directory holding the records. + #[must_use] + pub fn dir(&self) -> &Path { + &self.inner.dir + } + + /// The file the record at `address` is kept in. + #[must_use] + pub fn file_for(&self, address: &XorName) -> PathBuf { + self.path_for(address) + } + + /// Every state this node can serve, sorted by address. + /// + /// Taken from the index, which every other answer the store gives is also + /// based on. A record whose file has gone is left out: it is not something + /// this node can offer to anyone. + #[must_use] + pub fn held_states(&self) -> Vec { + let mut states: Vec = self + .inner + .index + .lock() + .values() + .filter(|entry| entry.on_disk) + .map(|entry| entry.state) + .collect(); + states.sort_unstable_by_key(|state| state.address); + states + } + + /// What the store has done since it was opened. + #[must_use] + pub fn stats(&self) -> PointerStoreStats { + self.inner.counters.snapshot() + } + + /// Remove the record at `address` and forget it entirely. + /// + /// For a node that is no longer responsible for the address. Unlike a + /// record lost to the disk, nothing about it is remembered, because the + /// node is not meant to take it back. Returns whether a file was removed, + /// so the caller can give its space back to the disk budget. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists and cannot be removed; the + /// record is then still held and still indexed. + pub async fn delete(&self, address: &XorName) -> Result { + let inner = Arc::clone(&self.inner); + let address = *address; + let removed = spawn_blocking(move || inner.delete_blocking(&address)) + .await + .map_err(|e| Error::Storage(format!("pointer delete panicked: {e}")))??; + if removed { + self.inner.counters.deleted.fetch_add(1, Ordering::Relaxed); + } + Ok(removed) + } + + /// The state this node can actually serve at `address`, having read it + /// back. + /// + /// The index is a claim about a file; this is that claim checked. The file + /// must still be there and must still be the record the index names — + /// comparing the state, not merely the address, because the disk and the + /// index are updated under one lock and an answer taken between the two + /// would otherwise describe a record that is no longer the one held. + /// + /// Only the structure is parsed: these bytes verified when they were + /// committed, and the question here is what is held, not whether it is + /// authentic. Corruption inside the signature therefore passes here — the + /// state is read from the body — and is caught by [`Self::get`], which + /// verifies and disowns. Checking it here instead would put an ML-DSA + /// verification in front of the payment gate, which is the one place it + /// must not be. A failed check disowns the entry, so the arrival that found + /// it becomes a repair. + fn held_state(&self, address: &XorName) -> Option { + let entry = self.snapshot(address).filter(|entry| entry.on_disk)?; + let serves = matches!( + read_record_file(&self.path_for(address)), + Ok(Some(ref bytes)) if matches!( + PointerState::parse(bytes), + Ok(ref state) if *state == entry.state + ) + ); + if !serves { + self.disown_if_unchanged(address, Some(entry.generation)); + return None; + } + Some(entry.state) + } + + /// Copy out what is held for `address`, releasing the index lock at once. + fn snapshot(&self, address: &XorName) -> Option { + self.inner.index.lock().get(address).copied() + } + + /// Stop serving `address`, but only if the entry is still the exact one + /// the caller found unreadable. + /// + /// The entry stays, marked as no longer on disk: what was lost is what says + /// which states can restore it. + /// + /// A read is not atomic with a write. Removing unconditionally would let a + /// slow read of a corrupt file erase the entry for a record committed while + /// it was reading, leaving the node holding a record it no longer + /// announces, commits to, or can be audited for. Matching on the generation + /// rather than the state identifier also covers the case where the record + /// written meanwhile is a *repair of the same state*, which a state + /// comparison could not tell apart from the entry being disowned. + fn disown_if_unchanged(&self, address: &XorName, claimed: Option) { + let Some(claimed) = claimed else { + // Nothing was claimed when the read began, so there is nothing this + // read is entitled to disown. + return; + }; + let mut index = self.inner.index.lock(); + if let Some(entry) = index.get_mut(address) { + if entry.generation == claimed { + entry.on_disk = false; + } + } + } + + /// Path of the file backing `address`. + fn path_for(&self, address: &XorName) -> PathBuf { + record_path(&self.inner.dir, address) + } +} + +impl Inner { + /// Keep `bytes` as the record just replaced at `address`, dropping what + /// has aged out and, past the cap, the oldest. + fn keep_superseded(&self, address: XorName, bytes: Vec) { + let now = Instant::now(); + let mut superseded = self.superseded.lock(); + superseded.retain(|_, (at, _)| now.duration_since(*at) < SUPERSEDED_RETENTION); + while superseded.len() >= MAX_SUPERSEDED { + let Some(oldest) = superseded + .iter() + .min_by_key(|(_, (at, _))| *at) + .map(|(address, _)| *address) + else { + break; + }; + superseded.remove(&oldest); + } + superseded.insert(address, (now, bytes)); + } + + /// Remove the file and the index entry for `address` under one lock, so a + /// commit can never land between the two and be forgotten on disk. + fn delete_blocking(&self, address: &XorName) -> Result { + let shard = shard_dir(&self.dir, address); + let path = record_path(&self.dir, address); + let removed = { + let mut index = self.index.lock(); + let removed = match std::fs::remove_file(&path) { + Ok(()) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "cannot delete {}: {e}", + path.display() + ))) + } + }; + index.remove(address); + removed + }; + if removed { + if let Err(e) = sync_directory(&shard) { + warn!( + "{} is deleted, but {} could not be synced: {e}", + path.display(), + shard.display() + ); + } + } + Ok(removed) + } + + /// Compare, write and re-index under one lock. + /// + /// Runs entirely inside a blocking task: the lock is synchronous and is + /// never released between the decision and the index update, so neither + /// caller cancellation nor a concurrent writer can separate them. + // Holding the guard across the write is the point of this function, so the + // lint's advice to drop it earlier would reintroduce exactly the window + // this closes: a second writer deciding against an index that no longer + // describes the disk. + #[allow( + clippy::significant_drop_tightening, + reason = "the write must happen under the same guard as the decision" + )] + fn commit_blocking( + &self, + record: &Pointer, + reservation: Option, + ) -> Result { + let address = record.address(); + let shard = shard_dir(&self.dir, &address); + let path = record_path(&self.dir, &address); + + // A cheap look before doing any work. The authoritative check is the + // one under the lock below; this only avoids staging a file for an + // arrival that is already obviously a no-op. + if let Some(entry) = self.index.lock().get(&address).filter(|e| e.on_disk) { + if entry.state.state_id == record.state_id() { + return Ok(PutOutcome::Unchanged); + } + if !record.state().replaces(&entry.state) { + return Ok(PutOutcome::Stale); + } + } + + // Stage and fsync the new bytes *before* taking the lock. This is the + // slow part — a 5 KB write plus an fsync — and holding the index lock + // across it would block every other address, including the cheap + // lookups async callers make. + let seq = self.write_seq.fetch_add(1, Ordering::Relaxed); + let temp = shard.join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); + if let Err(failed) = stage(&temp, &record.to_bytes()) { + // Bytes that could not be cleaned up are still on the disk, so the + // charge for them stands rather than going back. + if failed.bytes_remain { + settle(reservation); + } + return Err(failed.error); + } + + let outcome = { + let mut index = self.index.lock(); + + // Re-check: staging is not instantaneous and a newer state may + // have committed while it ran. + let replacing = index.get(&address).is_some_and(|entry| entry.on_disk); + let outcome = match index.get(&address) { + // Nothing held, or a record this node lost: either way the + // write must happen, whatever state it carries. + None | Some(IndexEntry { on_disk: false, .. }) => PutOutcome::Changed, + Some(entry) if entry.state.state_id == record.state_id() => PutOutcome::Unchanged, + Some(entry) if record.state().replaces(&entry.state) => PutOutcome::Changed, + Some(_) => PutOutcome::Stale, + }; + if outcome != PutOutcome::Changed { + // Nothing lands, so the charge goes back — unless the staged + // bytes could not be removed, in which case they are still on + // the disk and the charge has to stand for them. + if std::fs::remove_file(&temp).is_err() { + settle(reservation); + } + return Ok(outcome); + } + + // What this replaces, read under the lock so it is the record the + // index names. Kept for an audit that bound it; a record that + // cannot be read is not kept, and an audit owed it fails as it + // would have on the lost file. + let previous = if replacing { + read_record_file(&path).ok().flatten() + } else { + None + }; + + // The rename is the commit point: nothing fallible happens between + // it and the index update, and both are under this one lock. + if let Err(e) = rename_with_retry(&temp, &path) { + if std::fs::remove_file(&temp).is_err() { + settle(reservation); + } + return Err(Error::Storage(format!( + "cannot rename {} onto {}: {e}", + temp.display(), + path.display() + ))); + } + let generation = self.generation.fetch_add(1, Ordering::Relaxed); + index.insert(address, IndexEntry::of(record, generation)); + if let Some(previous) = previous { + self.keep_superseded(address, previous); + } + // A file is on the disk now. Charge it whether or not this replaced + // one: telling those apart would mean trusting an observation taken + // before the rename, and that observation can be wrong in the one + // direction that matters — a file the index claims can be gone, so + // what looks like a replacement grows the disk after all. + // Over-counting corrects itself at the next measurement. + settle(reservation); + outcome + }; + + // Durability of the directory entry, after the commit and outside the + // lock. The record is already stored and indexed, so a failure here + // cannot be reported as "nothing happened"; it is logged instead, which + // is how an operator learns the filesystem is not giving the store what + // it asks for. + if let Err(e) = sync_directory(&shard) { + warn!( + "{} is stored and indexed, but {} could not be synced: {e}. It is \ + visible now; its survival across a power loss depends on the \ + filesystem", + path.display(), + shard.display() + ); + } + Ok(outcome) + } +} + +/// Take the exclusive lock that grants use of `dir`. +fn acquire_lock(dir: &Path) -> Result { + let path = dir.join(LOCK_FILE_NAME); + let file = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + .map_err(|e| { + Error::Storage(format!( + "cannot create the pointer store lock {}: {e}", + path.display() + )) + })?; + file.try_lock_exclusive().map_err(|e| { + Error::Storage(format!( + "another pointer store already has {} open ({e}). Two stores over one \ + directory each keep their own index, and the staler one would overwrite \ + the other's newer record", + dir.display() + )) + })?; + Ok(file) +} + +/// Read one record file, refusing anything that is not exactly a record long. +/// +/// The length is taken from the directory entry before any bytes are read, so a +/// corrupt oversized file cannot be pulled into memory. +fn read_record_file(path: &Path) -> Result>> { + let file = match File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(Error::Storage(format!("pointer read failed: {e}"))), + }; + let len = file + .metadata() + .map_err(|e| Error::Storage(format!("pointer stat failed: {e}")))? + .len(); + if len != POINTER_WIRE_LEN as u64 { + warn!( + "Pointer file {} is {len} bytes, not {POINTER_WIRE_LEN}", + path.display() + ); + return Ok(None); + } + + let mut bytes = Vec::with_capacity(POINTER_WIRE_LEN); + // One byte past the record: a file that grew between the stat and the read + // is refused rather than silently truncated into something that parses. + file.take(POINTER_WIRE_LEN as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|e| Error::Storage(format!("pointer read failed: {e}")))?; + if bytes.len() != POINTER_WIRE_LEN { + warn!( + "Pointer file {} changed size while being read", + path.display() + ); + return Ok(None); + } + Ok(Some(bytes)) +} + +/// Create the shard directories, so a write never has to. +fn create_shards(dir: &Path) -> Result<()> { + for shard in 0..SHARD_COUNT { + let path = dir.join(format!("{shard:02x}")); + std::fs::create_dir_all(&path) + .map_err(|e| Error::Storage(format!("cannot create {}: {e}", path.display())))?; + } + sync_directory(dir) +} + +/// Rebuild the index from the records in `dir`'s shards. +/// +/// Each record's structure is parsed, but its signature is not checked. Every +/// record was verified when it was committed, and every read verifies it again +/// before serving it, so a file damaged in place is caught — and disowned — the +/// first time anyone asks for it. Checking an ML-DSA signature per record here +/// instead would make opening a large store take minutes, and the chunk store +/// likewise opens on names alone. +/// +/// A record left at the top level by the flat layout earlier builds used is +/// moved into its shard. +fn scan(dir: &Path) -> Result> { + let entries = std::fs::read_dir(dir) + .map_err(|e| Error::Storage(format!("cannot read {}: {e}", dir.display())))?; + + let mut index = HashMap::new(); + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + warn!("Skipping unreadable pointer directory entry: {e}"); + continue; + } + }; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name == LOCK_FILE_NAME { + continue; + } + if path.is_dir() { + if name.len() == 2 && name.bytes().all(|b| b.is_ascii_hexdigit()) { + scan_shard(&path, &mut index); + } + continue; + } + adopt_flat_record(dir, &path, name, &mut index); + } + Ok(index) +} + +/// Index the records in one shard directory. +fn scan_shard(shard: &Path, index: &mut HashMap) { + let entries = match std::fs::read_dir(shard) { + Ok(entries) => entries, + Err(e) => { + warn!("Skipping unreadable pointer shard {}: {e}", shard.display()); + return; + } + }; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + warn!("Skipping unreadable pointer directory entry: {e}"); + continue; + } + }; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name.starts_with(TEMP_PREFIX) { + // A temporary file means a crash mid-write: the rename never + // happened, so the old record (if any) is intact and the partial + // file has no claim. Sweep it, or the next write to that address + // collides with it — `create_new` would refuse, and the write + // sequence restarts at zero on every opening. + if let Err(e) = std::fs::remove_file(&path) { + warn!( + "Could not sweep the partial pointer write {}: {e}", + path.display() + ); + } + continue; + } + let Some(state) = parse_record_file(&path) else { + continue; + }; + let in_its_shard = shard + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n == shard_name(&state.address)); + if hex::encode(state.address) != name || !in_its_shard { + warn!( + "Skipping pointer file {} that is not at its own address", + path.display() + ); + continue; + } + let generation = u64::try_from(index.len()).unwrap_or(u64::MAX); + index.insert(state.address, IndexEntry::held(state, generation)); + } +} + +/// Move a record from the flat layout into its shard and index it. +fn adopt_flat_record( + dir: &Path, + path: &Path, + name: &str, + index: &mut HashMap, +) { + if name.starts_with(TEMP_PREFIX) { + if let Err(e) = std::fs::remove_file(path) { + warn!( + "Could not sweep the partial pointer write {}: {e}", + path.display() + ); + } + return; + } + let Some(state) = parse_record_file(path) else { + return; + }; + if hex::encode(state.address) != name { + warn!( + "Skipping pointer file {} that is not at its own address", + path.display() + ); + return; + } + let target = record_path(dir, &state.address); + if let Err(e) = rename_with_retry(path, &target) { + warn!( + "Could not move pointer file {} into its shard: {e}", + path.display() + ); + return; + } + let generation = u64::try_from(index.len()).unwrap_or(u64::MAX); + index.insert(state.address, IndexEntry::held(state, generation)); +} + +/// Read a record file and parse its structure, logging anything unusable. +fn parse_record_file(path: &Path) -> Option { + let bytes = match read_record_file(path) { + Ok(Some(bytes)) => bytes, + Ok(None) => return None, + Err(e) => { + warn!("Skipping unreadable pointer file {}: {e}", path.display()); + return None; + } + }; + match PointerState::parse(&bytes) { + Ok(state) => Some(state), + Err(e) => { + warn!("Skipping invalid pointer file {}: {e}", path.display()); + None + } + } +} + +/// Write `bytes` to `temp` and fsync it, ready to be renamed into place. +/// +/// Leaves nothing half written: the bytes are durable in the temporary file +/// before any rename can make them visible. The caller performs the rename, +/// which is the commit point. +/// +/// A failure tries to remove what it wrote, and says whether that worked: +/// `Err(StagingFailed { bytes_remain })` reports a partial file still on the +/// disk, so the caller knows the charge for it cannot be given back. +fn stage(temp: &Path, bytes: &[u8]) -> std::result::Result<(), StagingFailed> { + // `create_new` so a leftover temporary file from a crashed write is never + // silently appended to or shared with a concurrent writer. + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp) + .map_err(|e| StagingFailed { + // Nothing was created, so nothing is left behind. + error: Error::Storage(format!("cannot create {}: {e}", temp.display())), + bytes_remain: false, + })?; + let written = file + .write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|e| Error::Storage(format!("cannot write {}: {e}", temp.display()))); + drop(file); + if let Err(error) = written { + // A partial file exists. If it cannot be removed it is still occupying + // the disk, and its charge has to stand for it. + return Err(StagingFailed { + error, + bytes_remain: std::fs::remove_file(temp).is_err(), + }); + } + Ok(()) +} + +/// A staged write that did not complete, and whether it left bytes behind. +struct StagingFailed { + /// What went wrong, for the caller to return. + error: Error, + /// Whether a partial file is still on the disk. + bytes_remain: bool, +} + +/// Turn a charge into bytes that are now on the disk. +/// +/// A charge that is simply dropped is released instead, which is what every +/// path that writes nothing wants. +fn settle(reservation: Option) { + if let Some(reservation) = reservation { + reservation.commit(); + } +} + +/// Flush the directory entry a rename created. +/// +/// Without it a crash can leave the entry unflushed and the record invisible on +/// restart. +#[cfg(unix)] +fn sync_directory(dir: &Path) -> Result<()> { + match File::open(dir) { + Ok(handle) => handle + .sync_all() + .map_err(|e| Error::Storage(format!("cannot sync {}: {e}", dir.display()))), + Err(e) => Err(Error::Storage(format!( + "cannot open {} to flush it: {e}", + dir.display() + ))), + } +} + +/// As above, where there is no such thing to ask for. +/// +/// A directory cannot be opened as a file on Windows without backup semantics, +/// so the Unix form fails on every write there — which is not a durability +/// warning, it is the wrong question. NTFS orders the rename's own metadata. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps, reason = "one signature for both platforms")] +fn sync_directory(_dir: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + reason = "test assertions" +)] +mod tests { + use super::*; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind, POINTER_BODY_LEN}; + use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; + use std::future::Future; + + fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) + } + + fn signed(seed: u8, counter: u64, target_byte: u8) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(PointerTargetKind::Chunk, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + /// A record with its signature destroyed: the body still parses, the + /// record does not verify. + fn forged(record: &Pointer) -> Vec { + let mut bytes = record.to_bytes(); + if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 3) { + *byte ^= 0xff; + } + assert!( + Pointer::from_bytes(&bytes).is_err(), + "the forged record must not verify" + ); + bytes + } + + async fn store() -> (PointerStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("open store"); + (store, dir) + } + + /// The record an update replaces stays servable for a while, byte for + /// byte, so an audit that bound it before the update is not failed by it. + #[tokio::test] + async fn an_update_keeps_the_record_it_replaced_for_a_while() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + store.put_bytes(&first.to_bytes()).await.expect("put"); + assert_eq!( + store.superseded(&first.address()), + None, + "a creation replaces nothing" + ); + + let second = signed(1, 2, 2); + assert_eq!( + store.put_bytes(&second.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert_eq!( + store.superseded(&first.address()), + Some(first.to_bytes()), + "the replaced record, exactly as it was held" + ); + assert_eq!( + store.record_bytes(&first.address()).await.expect("read"), + Some(second.to_bytes()), + "and the new one is what is held" + ); + + // A stale arrival replaces nothing, so it keeps nothing. + store.put_bytes(&first.to_bytes()).await.expect("put"); + assert_eq!(store.superseded(&first.address()), Some(first.to_bytes())); + } + + #[tokio::test] + async fn stores_then_reads_back_the_same_bytes() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + assert_eq!( + store.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + + let read = store + .get(&record.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(read.to_bytes(), record.to_bytes()); + assert_eq!(store.state_id(&record.address()), Some(record.state_id())); + assert!(store.contains(&record.address())); + assert_eq!(store.len(), 1); + } + + #[tokio::test] + async fn a_higher_counter_replaces_and_a_lower_one_does_not() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + let second = signed(1, 2, 1); + assert_eq!( + store.put_bytes(&first.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert_eq!( + store.put_bytes(&second.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert_eq!( + store.put_bytes(&first.to_bytes()).await.expect("put"), + PutOutcome::Stale + ); + + let held = store + .get(&first.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 2); + assert_eq!(store.len(), 1, "an update replaces in place"); + } + + #[tokio::test] + async fn re_signing_one_state_never_reaches_the_disk() { + // One payment must not fund many writes. Every re-signature of a stored + // state is a no-op, so storage, replication and commitment work is paid + // for once. + let (store, _dir) = store().await; + let first = signed(1, 5, 5); + store.put_bytes(&first.to_bytes()).await.expect("put"); + + let path = store.file_for(&first.address()); + let held_bytes = std::fs::read(&path).expect("read"); + + for _ in 0..16 { + let variant = signed(1, 5, 5); + assert_ne!( + variant.to_bytes(), + first.to_bytes(), + "signing is randomized" + ); + assert_eq!(variant.state_id(), first.state_id()); + assert_eq!( + store.put_bytes(&variant.to_bytes()).await.expect("put"), + PutOutcome::Unchanged + ); + } + + assert_eq!( + std::fs::read(&path).expect("read"), + held_bytes, + "not one of the 16 re-signatures reached the disk" + ); + } + + #[tokio::test] + async fn a_resubmission_is_refused_before_its_signature_is_checked() { + // An arrival that cannot change anything is a no-op whether or not it + // is correctly signed, so it must not buy an ML-DSA verification. A + // record whose signature is destroyed proves the check was skipped: + // had it run, this would have been an error rather than Unchanged. + let (store, _dir) = store().await; + let held = signed(1, 4, 4); + store.put_bytes(&held.to_bytes()).await.expect("put"); + + assert_eq!( + store.put_bytes(&forged(&held)).await.expect("put"), + PutOutcome::Unchanged + ); + + let after = store + .get(&held.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(after.to_bytes(), held.to_bytes(), "nothing was written"); + } + + #[tokio::test] + async fn a_stale_arrival_is_refused_before_its_signature_is_checked() { + let (store, _dir) = store().await; + store + .put_bytes(&signed(1, 9, 1).to_bytes()) + .await + .expect("put"); + + // Unsigned *and* stale: refused as Stale, which can only happen if the + // signature was never checked. + let older = signed(1, 2, 1); + assert_eq!( + store.put_bytes(&forged(&older)).await.expect("put"), + PutOutcome::Stale + ); + } + + #[tokio::test] + async fn a_would_be_winner_is_always_verified() { + // Losing records skip verification; a record that would win never does. + let (store, _dir) = store().await; + store + .put_bytes(&signed(1, 1, 1).to_bytes()) + .await + .expect("put"); + + let winner = signed(1, 5, 1); + assert!(store.put_bytes(&forged(&winner)).await.is_err()); + + let held = store + .get(&winner.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 1, "the forged winner did not land"); + } + + #[tokio::test] + async fn an_arrival_that_changes_nothing_is_not_a_candidate() { + let (store, _dir) = store().await; + let held = signed(1, 6, 6); + store.put_bytes(&held.to_bytes()).await.expect("put"); + + match store.inspect(&held.to_bytes()).await.expect("inspect") { + Inspected::Unchanged(_) => (), + other => panic!("an identical state is not a candidate, got {other:?}"), + } + match store + .inspect(&signed(1, 1, 6).to_bytes()) + .await + .expect("inspect") + { + Inspected::Stale(_) => (), + other => panic!("a stale record is not a candidate, got {other:?}"), + } + } + + #[tokio::test] + async fn a_verified_record_exposes_what_a_payment_check_needs() { + let (store, _dir) = store().await; + let record = signed(1, 2, 2); + let verified = verified(&store, &record).await; + // A verified record is a `Pointer`, so the payment check reads the + // address and state straight off it — there is no wrapper type in + // between restating what it already knows. + assert_eq!(verified.address(), record.address()); + assert_eq!(verified.state_id(), record.state_id()); + assert_eq!(verified.to_bytes(), record.to_bytes()); + assert_eq!( + store.commit(verified, None).await.expect("commit"), + PutOutcome::Changed + ); + assert_eq!(store.len(), 1); + } + + #[tokio::test] + async fn a_commit_that_writes_nothing_gives_its_charge_back() { + // The reservation settles inside the commit transaction, so this is + // where a no-op has to release it. Losing a race is the reachable way + // to get there: the arrival is a candidate when it is inspected, and by + // the time it commits a better state is held. + // + // Asserted on the guard's own counters, because "the next reservation + // still succeeds" would pass just as well with the charge stranded. + let (store, dir) = store().await; + let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + + // Verified while nothing is held, so it is a candidate... + let slow = verified(&store, &signed(1, 2, 1)).await; + // ...and a better state lands before it commits. + store + .put_bytes(&signed(1, 7, 1).to_bytes()) + .await + .expect("put"); + + let before = chunks.capacity_counters(); + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + assert!( + chunks.capacity_counters().1 > before.1, + "taking a charge must raise in-flight" + ); + + assert_eq!( + store.commit(slow, Some(charge)).await.expect("commit"), + PutOutcome::Stale, + "the newer state must stand" + ); + assert_eq!( + chunks.capacity_counters(), + before, + "a write that did not happen must leave both counters where it found them" + ); + } + + #[tokio::test] + async fn a_commit_that_writes_charges_the_disk_for_it() { + // The other half: bytes that land move from in-flight to written, so + // the guard counts them against the reserve from then on. + let (store, dir) = store().await; + let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + + let record = verified(&store, &signed(1, 0, 1)).await; + // Exactly one allocation charge, not merely "more than before": a + // record is rounded up to the allocation unit, so an inequality would + // accept one charge standing in for two writes. + let one = crate::storage::ChunkStore::capacity_charge_for(POINTER_WIRE_LEN as u64); + let (written_before, in_flight_before) = chunks.capacity_counters(); + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + + assert_eq!( + store.commit(record, Some(charge)).await.expect("commit"), + PutOutcome::Changed + ); + assert_eq!( + chunks.capacity_counters(), + (written_before + one, in_flight_before), + "the bytes that landed move from in flight to written, exactly once" + ); + + // A replacement charges too: telling it apart would mean trusting an + // observation taken before the rename. + let update = verified(&store, &signed(1, 1, 1)).await; + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + assert_eq!( + store.commit(update, Some(charge)).await.expect("commit"), + PutOutcome::Changed + ); + assert_eq!( + chunks.capacity_counters(), + (written_before + 2 * one, in_flight_before) + ); + } + + #[tokio::test] + async fn a_commit_rechecks_what_verification_saw() { + // Verification runs before the payment check; a newer state can land + // while that check is in flight, and must not then be overwritten. + let (store, _dir) = store().await; + let slow = verified(&store, &signed(1, 2, 1)).await; + + // Someone else's newer state arrives while the payment is being checked. + store + .put_bytes(&signed(1, 7, 1).to_bytes()) + .await + .expect("put"); + + assert_eq!( + store.commit(slow, None).await.expect("commit"), + PutOutcome::Stale, + "the newer state must survive a late commit" + ); + let held = store + .get(&signed(1, 2, 1).address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 7); + } + + #[tokio::test] + async fn put_bytes_rejects_junk() { + let (store, _dir) = store().await; + assert!(store.put_bytes(b"not a pointer").await.is_err()); + assert!(store.put_bytes(&[]).await.is_err()); + assert!(store.is_empty()); + } + + #[tokio::test] + async fn every_rotation_of_a_delivery_reaches_one_answer() { + let records = [ + signed(1, 1, 9), + signed(1, 3, 4), + signed(1, 3, 1), + signed(1, 2, 8), + signed(1, 3, 7), + ]; + let address = records.first().expect("non-empty").address(); + + let mut winners = Vec::new(); + for rotation in 0..records.len() { + let (store, _dir) = store().await; + for offset in 0..records.len() { + let index = (rotation + offset) % records.len(); + let record = records.get(index).expect("in range"); + store.put_bytes(&record.to_bytes()).await.expect("put"); + } + let held = store.get(&address).await.expect("get").expect("present"); + winners.push(held.state_id()); + } + + let distinct: std::collections::BTreeSet<&XorName> = winners.iter().collect(); + assert_eq!( + distinct.len(), + 1, + "every rotation converges: {winners:?}. Exhaustive permutations are \ + covered by the convergence property test" + ); + } + + #[tokio::test] + async fn a_node_started_empty_reaches_the_same_answer() { + let dir = tempfile::tempdir().expect("tempdir"); + let address; + let before; + { + let store = PointerStore::new(dir.path()).await.expect("open"); + for record in [signed(1, 1, 1), signed(1, 4, 2), signed(1, 2, 3)] { + store.put_bytes(&record.to_bytes()).await.expect("put"); + } + address = signed(1, 1, 1).address(); + before = store.get(&address).await.expect("get").expect("present"); + } + + // A fresh store over the same directory, as after a restart. The first + // must be dropped: the directory lock allows only one at a time. + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!(reopened.len(), 1); + let after = reopened.get(&address).await.expect("get").expect("present"); + assert_eq!(after.to_bytes(), before.to_bytes()); + assert_eq!(reopened.state_id(&address), Some(before.state_id())); + } + + #[tokio::test] + async fn a_second_store_over_one_directory_is_refused() { + // Two indexes over one set of files means the staler one overwrites the + // fresher one's record. + let dir = tempfile::tempdir().expect("tempdir"); + let _first = PointerStore::new(dir.path()).await.expect("open"); + let err = PointerStore::new(dir.path()) + .await + .expect_err("a second store must not open the same directory"); + let message = format!("{err}"); + assert!( + message.contains("another pointer store already has"), + "it must be refused by the directory lock, not by something else: {message}" + ); + } + + #[tokio::test] + async fn two_owners_occupy_two_addresses() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + let second = signed(2, 1, 1); + assert_ne!(first.address(), second.address()); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.put_bytes(&second.to_bytes()).await.expect("put"); + assert_eq!(store.len(), 2); + } + + #[tokio::test] + async fn a_corrupt_file_is_forgotten_so_the_record_can_be_repaired() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + let path = store.file_for(&record.address()); + let mut bytes = std::fs::read(&path).expect("read back"); + if let Some(byte) = bytes.get_mut(10) { + *byte ^= 0xff; + } + std::fs::write(&path, &bytes).expect("corrupt it"); + + assert!(store.get(&record.address()).await.expect("get").is_none()); + assert!( + !store.contains(&record.address()), + "the index must not keep claiming a record the disk lost" + ); + assert_eq!(store.state_id(&record.address()), None); + + // The repair a peer would send is accepted rather than dismissed as + // "unchanged", which is the whole point of forgetting it. + assert_eq!( + store.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert!(store.get(&record.address()).await.expect("get").is_some()); + } + + /// Run the production sequence as far as the payment gate: inspect, then + /// verify the candidate it yields. + async fn verified(store: &PointerStore, record: &Pointer) -> Pointer { + match store.inspect(&record.to_bytes()).await.expect("inspect") { + Inspected::Candidate(parsed) => store.verify(parsed).await.expect("verify"), + other => panic!("expected a candidate, got {other:?}"), + } + } + + #[tokio::test] + async fn a_resubmission_repairs_a_record_the_disk_lost() { + // "Unchanged" is an acknowledgement: the sender stops on it. If the + // index still claims a record whose file has gone, answering Unchanged + // would end the write while this node holds nothing. + let (store, _dir) = store().await; + let record = signed(1, 0, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + // The file disappears under the node; the index has not noticed. + std::fs::remove_file(store.file_for(&record.address())).expect("remove"); + assert!( + store.contains(&record.address()), + "the index still claims it" + ); + + // The same state arriving again is a repair, not a no-op. + assert_eq!( + store.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed, + "a resubmission must repair a record the disk lost" + ); + assert!(store.get(&record.address()).await.expect("get").is_some()); + } + + #[tokio::test] + async fn a_lost_record_is_restored_and_cannot_be_rolled_back() { + // The node remembers what it lost. Without that the address would look + // untouched, where any record is admissible, and a replay of an older + // state would roll the node back. + let (store, _dir) = store().await; + for counter in 0..=3u64 { + store + .put_bytes(&signed(1, counter, 1).to_bytes()) + .await + .expect("put"); + } + let held = signed(1, 3, 1); + + std::fs::remove_file(store.file_for(&held.address())).expect("remove"); + // A read notices the loss and stops the node answering for it. + assert!(store.get(&held.address()).await.expect("get").is_none()); + assert!(!store.contains(&held.address()), "it is not served"); + assert_eq!(store.state_id(&held.address()), None); + + // What it lost is what it will take back, and so is anything newer. + assert!( + store.admits(&held.state()), + "the state this node lost must be admissible again" + ); + assert!( + store.admits(&signed(1, 9, 1).state()), + "so must a newer state the rest of the group has moved on to" + ); + assert!( + !store.admits(&signed(1, 2, 1).state()), + "but not one that loses to what was lost" + ); + + assert_eq!( + store.put_bytes(&held.to_bytes()).await.expect("put"), + PutOutcome::Changed, + "the repair must write, not be answered as unchanged" + ); + let back = store + .get(&held.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(back.counter(), 3); + assert_eq!(back.state_id(), held.state_id()); + + // Held again: any later counter is admitted, the held state is not. + assert!(store.admits(&signed(1, 4, 1).state())); + assert!(store.admits(&signed(1, 6, 1).state()), "a skip is admitted"); + assert!( + !store.admits(&held.state()), + "held again, so a resubmission" + ); + } + + #[tokio::test] + async fn a_file_swapped_for_another_valid_record_is_not_answered_for() { + // The index names one state; the disk holds a different, perfectly + // valid one. Answering from the index would acknowledge a state this + // node cannot serve. + let (store, _dir) = store().await; + let indexed = signed(1, 4, 1); + store.put_bytes(&indexed.to_bytes()).await.expect("put"); + + let other = signed(1, 9, 1); + std::fs::write(store.file_for(&indexed.address()), other.to_bytes()) + .expect("swap the file"); + + match store.inspect(&indexed.to_bytes()).await.expect("inspect") { + Inspected::Candidate(_) => (), + other => panic!("a state the node cannot serve must not be answered for: {other:?}"), + } + } + + #[tokio::test] + async fn a_deleted_file_is_forgotten_too() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + std::fs::remove_file(store.file_for(&record.address())).expect("remove"); + assert!(store.get(&record.address()).await.expect("get").is_none()); + assert!(!store.contains(&record.address())); + } + + #[tokio::test] + async fn a_scan_skips_what_it_cannot_validate() { + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(1, 1, 1); + { + let store = PointerStore::new(dir.path()).await.expect("open"); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + // Junk under a plausible name, an oversized file, and a partial write. + std::fs::write(store.file_for(&[9u8; 32]), b"not a pointer").expect("write junk"); + std::fs::write(store.file_for(&[8u8; 32]), vec![0u8; POINTER_WIRE_LEN * 4]) + .expect("write oversized"); + std::fs::write( + store.file_for(&[7u8; 32]).with_file_name(".tmp-abc-0"), + vec![0u8; POINTER_WIRE_LEN], + ) + .expect("write temp"); + } + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!(reopened.len(), 1, "only the valid record is indexed"); + assert!(reopened.contains(&record.address())); + } + + #[tokio::test] + async fn concurrent_writers_leave_the_index_agreeing_with_the_disk() { + let (store, _dir) = store().await; + let address = signed(1, 0, 0).address(); + + let mut tasks = Vec::new(); + for counter in 1..=12u64 { + let store = store.clone(); + let bytes = signed(1, counter, 1).to_bytes(); + tasks.push(tokio::spawn(async move { store.put_bytes(&bytes).await })); + } + for task in tasks { + task.await.expect("join").expect("put"); + } + + let held = store.get(&address).await.expect("get").expect("present"); + assert_eq!(held.counter(), 12, "the highest counter must win"); + assert_eq!(store.state_id(&address), Some(held.state_id())); + assert_eq!(store.len(), 1); + + // No temporary file survived the race. + assert!( + temporary_files(&store).is_empty(), + "temporary files were left behind" + ); + } + + /// Every temporary file anywhere under the store. + fn temporary_files(store: &PointerStore) -> Vec { + let mut found = Vec::new(); + for shard in std::fs::read_dir(store.dir()).expect("read dir").flatten() { + if !shard.path().is_dir() { + continue; + } + for entry in std::fs::read_dir(shard.path()) + .expect("read shard") + .flatten() + { + if entry.file_name().to_string_lossy().starts_with(TEMP_PREFIX) { + found.push(entry.path()); + } + } + } + found + } + + #[tokio::test] + async fn a_record_is_kept_in_the_shard_named_by_its_last_byte() { + let (store, _dir) = store().await; + let record = signed(3, 0, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + let path = store.file_for(&record.address()); + let last = record.address().last().copied().expect("32 bytes"); + assert_eq!( + path.parent().expect("a shard"), + store.dir().join(format!("{last:02x}")) + ); + assert_eq!( + std::fs::read(&path).expect("read"), + record.to_bytes(), + "the record is on disk where the store says it is" + ); + let shards = std::fs::read_dir(store.dir()) + .expect("read dir") + .flatten() + .filter(|e| e.path().is_dir()) + .count(); + assert_eq!(shards, 256, "every shard exists before any write needs it"); + } + + #[tokio::test] + async fn held_states_lists_what_is_served_in_address_order() { + let (store, _dir) = store().await; + let records: Vec = (1..=5u8).map(|seed| signed(seed, 2, seed)).collect(); + for record in &records { + store.put_bytes(&record.to_bytes()).await.expect("put"); + } + let lost = records.first().expect("five records"); + std::fs::remove_file(store.file_for(&lost.address())).expect("remove"); + assert!(store.get(&lost.address()).await.expect("get").is_none()); + + let held = store.held_states(); + let mut expected: Vec = records.iter().skip(1).map(Pointer::address).collect(); + expected.sort_unstable(); + assert_eq!( + held.iter().map(|state| state.address).collect::>(), + expected, + "sorted, and without the record whose file is gone" + ); + for state in &held { + assert_eq!(store.state_id(&state.address), Some(state.state_id)); + } + } + + #[tokio::test] + async fn delete_removes_the_record_and_forgets_it() { + let (store, _dir) = store().await; + let record = signed(4, 3, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + let path = store.file_for(&record.address()); + + assert!(store.delete(&record.address()).await.expect("delete")); + assert!(!path.exists(), "the file is gone"); + assert!(!store.contains(&record.address())); + assert!(store.get(&record.address()).await.expect("get").is_none()); + // Forgotten, not remembered as lost: an older state is admissible again, + // which a lost record would refuse. + assert!(store.admits(&signed(4, 1, 1).state())); + assert!( + !store.delete(&record.address()).await.expect("delete"), + "nothing left to delete" + ); + assert_eq!(store.stats().deleted, 1); + + // Still usable afterwards, and gone across a reopen too. + store + .put_bytes(&record.to_bytes()) + .await + .expect("put again"); + assert!(store.contains(&record.address())); + store.delete(&record.address()).await.expect("delete"); + let dir = store.dir().parent().expect("root").to_path_buf(); + drop(store); + let reopened = PointerStore::new(&dir).await.expect("reopen"); + assert!(!reopened.contains(&record.address())); + } + + #[tokio::test] + async fn stats_count_what_each_arrival_did() { + let (store, _dir) = store().await; + let first = signed(5, 1, 1); + let newer = signed(5, 2, 1); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.put_bytes(&newer.to_bytes()).await.expect("put"); + store + .put_bytes(&signed(5, 2, 1).to_bytes()) + .await + .expect("put"); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.get(&first.address()).await.expect("get"); + + assert_eq!( + store.stats(), + PointerStoreStats { + written: 2, + unchanged: 1, + stale: 1, + served: 1, + deleted: 0, + } + ); + } + + #[tokio::test] + async fn opening_indexes_records_without_verifying_them_and_a_read_still_does() { + // The scan parses structure only. A signature damaged in place is + // indexed, and caught by the first read, which disowns it. + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(6, 1, 1); + let path = { + let store = PointerStore::new(dir.path()).await.expect("open"); + store.put_bytes(&record.to_bytes()).await.expect("put"); + store.file_for(&record.address()) + }; + let mut bytes = std::fs::read(&path).expect("read"); + if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 7) { + *byte ^= 0xff; + } + std::fs::write(&path, &bytes).expect("damage"); + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert!( + reopened.contains(&record.address()), + "indexed on its structure" + ); + assert!( + reopened + .get(&record.address()) + .await + .expect("get") + .is_none(), + "never served" + ); + assert!( + !reopened.contains(&record.address()), + "and disowned once read" + ); + } + + #[tokio::test] + async fn a_record_in_the_flat_layout_is_moved_into_its_shard() { + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(7, 4, 2); + let flat = dir + .path() + .join(POINTERS_DIR_NAME) + .join(hex::encode(record.address())); + std::fs::create_dir_all(flat.parent().expect("parent")).expect("mkdir"); + std::fs::write(&flat, record.to_bytes()).expect("write"); + + let store = PointerStore::new(dir.path()).await.expect("open"); + assert!(!flat.exists(), "moved out of the top level"); + assert!(store.file_for(&record.address()).exists(), "into its shard"); + let held = store + .get(&record.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.state_id(), record.state_id()); + } + + #[tokio::test] + async fn a_cancelled_commit_leaves_the_index_agreeing_with_the_disk() { + // Dropping the caller's future must not split the transaction: either + // the write and the index update both happened, or neither did. + let (store, _dir) = store().await; + let record = signed(1, 3, 3); + let prepared = verified(&store, &record).await; + + { + let committing = store.commit(prepared, None); + tokio::pin!(committing); + // Poll exactly once, then drop. That first poll hands the write to + // a blocking thread and returns `Pending`, so the caller is always + // abandoned mid-commit. A timeout would not be: on a fast machine + // the commit finishes inside any timeout and the test then proves + // nothing, which is how this failed on CI and passed locally. + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + committing.as_mut().poll(&mut cx).is_pending(), + "the first poll must hand the write to a blocking task, not finish it" + ); + } + + // Wait for the detached task to settle rather than guessing at a + // duration: poll until the disk and the index agree and stop changing. + let mut settled = false; + for _ in 0..200 { + let on_disk = store.get(&record.address()).await.expect("get").is_some(); + if on_disk == store.contains(&record.address()) && on_disk { + settled = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let on_disk = store.get(&record.address()).await.expect("get").is_some(); + let indexed = store.contains(&record.address()); + assert_eq!( + on_disk, indexed, + "the index and the disk must agree however the commit was interrupted" + ); + assert!( + settled, + "the detached commit must run to completion rather than stopping half done" + ); + } + + #[tokio::test] + async fn an_empty_store_holds_nothing() { + let (store, _dir) = store().await; + assert!(store.is_empty()); + assert_eq!(store.state_id(&[0u8; 32]), None); + assert!(store.get(&[0u8; 32]).await.expect("get").is_none()); + } + + #[tokio::test] + async fn a_failed_read_does_not_erase_a_record_written_since() { + // A read that finds the file unreadable may only disown the entry it + // set out to read. A commit that lands meanwhile must survive, or the + // node ends up holding a record it never announces or commits to. + let (store, _dir) = store().await; + let old = signed(1, 1, 1); + store.put_bytes(&old.to_bytes()).await.expect("put"); + + // Simulate the interleaving: the read observed the old entry, then a + // newer state was committed, and only then does the read disown what + // it saw. + let observed = store.snapshot(&old.address()).map(|entry| entry.generation); + let new = signed(1, 2, 1); + store.put_bytes(&new.to_bytes()).await.expect("put"); + + store.disown_if_unchanged(&old.address(), observed); + assert_eq!( + store.state_id(&new.address()), + Some(new.state_id()), + "the record committed during the read must still be indexed" + ); + + // And the ordinary case still works: disowning what is actually there. + let current = store.snapshot(&new.address()).map(|entry| entry.generation); + store.disown_if_unchanged(&new.address(), current); + assert!(!store.contains(&new.address())); + } + + #[tokio::test] + async fn a_repair_of_the_same_state_is_not_erased_by_a_second_reader() { + // Two readers observe one corrupt entry. The first disowns it, a repair + // restores exactly the same logical state, and the second reader must + // not then erase the repair — the state identifier alone cannot tell + // the repaired entry from the corrupt one it replaced. + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + let first_reader = store + .snapshot(&record.address()) + .map(|entry| entry.generation); + let second_reader = first_reader; + + // Reader one finds the file unreadable and disowns what it saw. + store.disown_if_unchanged(&record.address(), first_reader); + assert!(!store.contains(&record.address())); + + // A peer repairs it with the very same state. + assert_eq!( + store.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + + // Reader two, still holding its stale observation, must not erase it. + store.disown_if_unchanged(&record.address(), second_reader); + assert!( + store.contains(&record.address()), + "the repair must survive a second reader disowning the old entry" + ); + assert_eq!(store.state_id(&record.address()), Some(record.state_id())); + } + + #[tokio::test] + async fn a_leftover_partial_write_is_swept_and_does_not_block_the_next_write() { + // The write sequence restarts at zero on every opening, so a temporary + // file left by a crash would collide with the next write to that + // address and `create_new` would refuse it. + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(1, 1, 1); + { + let store = PointerStore::new(dir.path()).await.expect("open"); + let leftover = store + .dir() + .join(format!("{TEMP_PREFIX}{}-0", hex::encode(record.address()))); + std::fs::write(&leftover, vec![0u8; POINTER_WIRE_LEN]).expect("write leftover"); + } + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!( + reopened.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed, + "a swept leftover must not block the first write to its address" + ); + } + + #[tokio::test] + async fn holding_the_key_is_not_holding_the_state() { + // The fetch decision a mutable record needs: a replica on version N + // must not count as satisfied for version N+1. + let (store, _dir) = store().await; + let old = signed(1, 1, 1); + let new = signed(1, 2, 1); + store.put_bytes(&old.to_bytes()).await.expect("put"); + + assert!(store.contains(&new.address())); + } +} diff --git a/src/replication/commitment.rs b/src/replication/commitment.rs index 61082510..714c2034 100644 --- a/src/replication/commitment.rs +++ b/src/replication/commitment.rs @@ -65,6 +65,27 @@ pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { *h.finalize().as_bytes() } +/// Key-derivation context for a pointer's commitment leaf (ADR-0016). +const POINTER_LEAF_CONTEXT: &str = "autonomi.pointer.commitment-leaf.v1"; + +/// The `bytes_hash` a pointer at `address` is committed under (ADR-0016). +/// +/// A chunk is committed as `(key, key)`: its address is the hash of its bytes. +/// A pointer's address is not, and its bytes change with every update, so it +/// is committed under a value derived from the address alone. The root then +/// moves only when a pointer is added or dropped, never on an update, and an +/// auditor tells the two kinds apart from the leaf itself: `bytes_hash == key` +/// for a chunk, `bytes_hash == pointer_leaf_hash(key)` for a pointer. Derive-key +/// keeps them apart — a chunk leaf cannot equal a pointer leaf without a BLAKE3 +/// preimage across modes. +/// +/// Possession of a pointer is proved in round 2 by serving the whole signed +/// record, which the auditor verifies; nothing about the bytes is bound here. +#[must_use] +pub fn pointer_leaf_hash(address: &XorName) -> [u8; 32] { + blake3::derive_key(POINTER_LEAF_CONTEXT, address) +} + /// Combine two child hashes into a Merkle internal-node hash. #[must_use] pub fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { @@ -106,6 +127,9 @@ pub struct MerkleTree { /// `levels[0].len() == leaves.len()`; `levels[L].len() == 1` where L /// is the root level. levels: Vec>, + /// Sorted indices of the leaves that commit a pointer, found once at build + /// so the audit and persistence paths never re-derive them per leaf. + pointer_leaves: Vec, } impl MerkleTree { @@ -137,6 +161,15 @@ impl MerkleTree { } } + // A chunk leaf is `(key, key)`, so the pointer check only runs on the + // leaves that are not. + let pointer_leaves: Vec = entries + .iter() + .enumerate() + .filter(|(_, (k, bh))| bh != k && *bh == pointer_leaf_hash(k)) + .map(|(idx, _)| idx) + .collect(); + let leaves: Vec<(XorName, [u8; 32])> = entries .into_iter() .map(|(k, bh)| { @@ -152,7 +185,11 @@ impl MerkleTree { levels.push(level.clone()); } - Ok(Self { leaves, levels }) + Ok(Self { + leaves, + levels, + pointer_leaves, + }) } /// The Merkle root of this tree. @@ -241,6 +278,34 @@ impl MerkleTree { self.leaves.get(idx).map(|(k, _)| *k) } + /// Whether the leaf at `idx` commits a pointer rather than a chunk. + #[must_use] + pub fn is_pointer_leaf(&self, idx: usize) -> bool { + self.pointer_leaves.binary_search(&idx).is_ok() + } + + /// Whether `key` is committed, as a pointer. + #[must_use] + pub fn commits_pointer(&self, key: &XorName) -> bool { + self.key_index(key) + .is_some_and(|idx| self.is_pointer_leaf(idx)) + } + + /// The keys committed as pointers, in the tree's sorted order. + #[must_use] + pub fn pointer_leaf_keys(&self) -> Vec { + self.pointer_leaves + .iter() + .filter_map(|idx| self.key_at(*idx)) + .collect() + } + + /// How many leaves commit a pointer. + #[must_use] + pub fn pointer_count(&self) -> usize { + self.pointer_leaves.len() + } + /// The sorted leaf index of `key`, if committed. `O(log n)` binary search /// over the (key-sorted) leaves — no separate key list needed, so callers /// don't have to keep a duplicate `sorted_keys` Vec alongside the tree. @@ -439,6 +504,30 @@ mod tests { assert!(matches!(result, Err(CommitmentError::EmptyKeySet))); } + /// A pointer leaf (ADR-0016) is told apart from a chunk leaf by its hash + /// alone, and a leaf hashed any other way is neither. + #[test] + fn pointer_leaves_are_told_apart_from_chunk_leaves() { + let chunk = [1u8; 32]; + let pointer = [2u8; 32]; + let other = [3u8; 32]; + let tree = MerkleTree::build(vec![ + (chunk, chunk), + (pointer, pointer_leaf_hash(&pointer)), + (other, [9u8; 32]), + ]) + .unwrap(); + assert!(tree.commits_pointer(&pointer)); + assert!(!tree.commits_pointer(&chunk)); + assert!(!tree.commits_pointer(&other)); + assert!( + !tree.commits_pointer(&[4u8; 32]), + "an uncommitted key is not" + ); + assert_eq!(tree.pointer_leaf_keys(), vec![pointer]); + assert_eq!(tree.pointer_count(), 1); + } + #[test] fn duplicate_keys_rejected() { let result = MerkleTree::build(vec![(xn(1), bh(1)), (xn(1), bh(2))]); diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 62b965f9..08d3ce46 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -28,7 +28,7 @@ //! `2 × (key_count × ~64 bytes + signature_size)` — for 10k keys, ~1.3 MB. use saorsa_core::identity::PeerId; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -38,8 +38,8 @@ use serde::{Deserialize, Serialize}; use crate::ant_protocol::XorName; use crate::replication::commitment::{ - commitment_hash, sign_commitment, verify_commitment_signature, CommitmentError, MerkleTree, - StorageCommitment, + commitment_hash, pointer_leaf_hash, sign_commitment, verify_commitment_signature, + CommitmentError, MerkleTree, StorageCommitment, }; /// Auditor-side per-peer commitment state. @@ -280,6 +280,12 @@ impl BuiltCommitment { self.tree.leaf_keys() } + /// The subset of [`Self::leaf_keys`] committed as pointers (ADR-0016). + #[must_use] + pub fn pointer_leaf_keys(&self) -> Vec { + self.tree.pointer_leaf_keys() + } + /// Reconstruct a `BuiltCommitment` from a persisted signed commitment and a /// `tree` rebuilt from its leaf keys — WITHOUT re-signing, so the pin /// (`commitment_hash`) is preserved exactly across a restart (ML-DSA @@ -362,14 +368,44 @@ struct PersistedSlot { /// slot as already expired. `None` if the slot was never gossiped — then it /// survives reload only while it is the current slot. expires_at_unix: Option, + /// The subset of `leaf_keys` committed as pointers (ADR-0016). A pointer + /// leaf is `(key, pointer_leaf_hash(key))`, not `(key, key)`, so without + /// this the rebuilt tree would miss the signed root and the slot be lost. + /// + /// Not part of this file: it is kept in the pointer-leaves sidecar (see + /// [`PersistedRetention::pointer_leaves_bytes`]), so the retention file + /// stays exactly what a release before pointers reads. + #[serde(skip)] + pointer_keys: Vec, } /// Persisted-format version. Bump on any layout OR semantic change so an /// incompatible on-disk snapshot is rejected (→ empty retention, which self-heals /// via re-gossip) rather than silently misinterpreted (e.g. an old field read /// under new semantics). +/// +/// Pointers did not change it. Their leaves are persisted beside this file, so +/// a node rolled back to a release that predates pointers still reloads every +/// slot it can answer for, and drops only the ones that commit a pointer, which +/// it could not answer for anyway. const RETENTION_FORMAT_VERSION: u32 = 1; +/// Version of the pointer-leaves sidecar. +const POINTER_LEAVES_FORMAT_VERSION: u32 = 1; + +/// The pointer leaves of each retained slot, keyed by the slot's commitment +/// hash (ADR-0016). +/// +/// Keyed by hash rather than by position so that a sidecar left behind by a +/// different snapshot, as after a rollback and upgrade, can only ever describe +/// the commitment it names: a hash binds the root, and the root binds which +/// leaves are pointers. +#[derive(Serialize, Deserialize)] +struct PersistedPointerLeaves { + version: u32, + slots: Vec<([u8; 32], Vec)>, +} + /// The persisted responder retention. Slots are newest-first; `has_current` /// says whether `slots[0]` was the live advertised commitment. #[derive(Serialize, Deserialize)] @@ -397,6 +433,48 @@ impl PersistedRetention { let this: Self = postcard::from_bytes(bytes).ok()?; (this.version == RETENTION_FORMAT_VERSION).then_some(this) } + + /// Serialize the pointer leaves of every slot, for the sidecar written + /// beside the retention file. `None` on a serialization error. + #[must_use] + pub fn pointer_leaves_bytes(&self) -> Option> { + let slots = self + .slots + .iter() + .filter(|slot| !slot.pointer_keys.is_empty()) + .filter_map(|slot| { + commitment_hash(&slot.commitment).map(|hash| (hash, slot.pointer_keys.clone())) + }) + .collect(); + postcard::to_allocvec(&PersistedPointerLeaves { + version: POINTER_LEAVES_FORMAT_VERSION, + slots, + }) + .ok() + } + + /// Give each slot the pointer leaves the sidecar records for its + /// commitment. Returns `false`, attaching nothing, on a corrupt sidecar or + /// an unknown version; a slot left without its pointer leaves then fails to + /// rebuild and is dropped, which fails open locally as a corrupt snapshot + /// does. + pub fn attach_pointer_leaves(&mut self, bytes: &[u8]) -> bool { + let Ok(sidecar) = postcard::from_bytes::(bytes) else { + return false; + }; + if sidecar.version != POINTER_LEAVES_FORMAT_VERSION { + return false; + } + let by_hash: HashMap<[u8; 32], Vec> = sidecar.slots.into_iter().collect(); + for slot in &mut self.slots { + if let Some(keys) = + commitment_hash(&slot.commitment).and_then(|hash| by_hash.get(&hash)) + { + slot.pointer_keys.clone_from(keys); + } + } + true + } } /// Responder retention state (ADR-0002). @@ -769,6 +847,7 @@ impl ResponderCommitmentState { commitment: c.commitment().clone(), leaf_keys: c.leaf_keys(), expires_at_unix, + pointer_keys: c.pointer_leaf_keys(), } }) .collect(); @@ -796,7 +875,18 @@ impl ResponderCommitmentState { // else a later slot would be wrongly promoted to current. let mut first_slot_restored = false; for (i, slot) in persisted.slots.iter().enumerate() { - let entries: Vec<_> = slot.leaf_keys.iter().map(|k| (*k, *k)).collect(); + let pointer_keys: HashSet<&XorName> = slot.pointer_keys.iter().collect(); + let entries: Vec<_> = slot + .leaf_keys + .iter() + .map(|k| { + if pointer_keys.contains(k) { + (*k, pointer_leaf_hash(k)) + } else { + (*k, *k) + } + }) + .collect(); let Ok(tree) = MerkleTree::build(entries) else { continue; }; @@ -1051,6 +1141,114 @@ mod tests { ); } + /// Build a signed commitment over `chunks` and `pointers`. + fn mixed(chunks: &[u8], pointers: &[u8]) -> BuiltCommitment { + let (pk, sk) = keypair(); + let mut entries: Vec<_> = chunks.iter().map(|i| (key(*i), key(*i))).collect(); + entries.extend( + pointers + .iter() + .map(|i| (key(*i), pointer_leaf_hash(&key(*i)))), + ); + BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk.to_bytes()).unwrap() + } + + /// A commitment holding pointers (ADR-0016) survives a restart like any + /// other. A pointer leaf is not `(key, key)`, so a snapshot that kept only + /// the key set would rebuild a different root, fail the signature check and + /// drop the slot, and every peer pinning it would then fail this node. + #[test] + fn a_commitment_holding_pointers_survives_a_restart() { + let built = mixed(&[1, 2, 3, 4], &[10, 11, 12]); + let pin = built.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(built); + state.mark_gossiped(pin); + + let snapshot = state.snapshot(); + let bytes = snapshot.to_bytes().expect("serialize"); + let sidecar = snapshot.pointer_leaves_bytes().expect("serialize sidecar"); + let mut reloaded = PersistedRetention::from_bytes(&bytes).expect("deserialize"); + assert!(reloaded.attach_pointer_leaves(&sidecar)); + let fresh = ResponderCommitmentState::new(); + fresh.restore(&reloaded); + + let got = fresh.lookup_by_hash(&pin).expect("pin survives restart"); + assert_eq!(got.hash(), pin); + assert!(fresh.is_held(&key(11)), "a committed pointer is still held"); + assert!(fresh.is_held(&key(2)), "a committed chunk is still held"); + assert_eq!( + got.pointer_leaf_keys(), + vec![key(10), key(11), key(12)], + "the pointers are still told apart from the chunks" + ); + } + + /// A node rolled back to a release that predates pointers keeps every + /// commitment it can still answer for. The retention file is exactly the + /// format such a release reads; it just knows nothing of the sidecar, so + /// the slot committing pointers is dropped and the chunk-only one a peer + /// pinned before the upgrade is not. + #[test] + fn a_rollback_keeps_every_chunk_only_commitment() { + let legacy = mixed(&[1, 2, 3], &[]); + let legacy_pin = legacy.hash(); + let current = mixed(&[1, 2, 3], &[10]); + let current_pin = current.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(legacy); + state.mark_gossiped(legacy_pin); + state.rotate(current); + state.mark_gossiped(current_pin); + + let bytes = state.snapshot().to_bytes().expect("serialize"); + assert_eq!( + postcard::take_from_bytes::(&bytes) + .map(|(v, _)| v) + .ok(), + Some(RETENTION_FORMAT_VERSION) + ); + // What an older release does: read the file and nothing beside it. + let older = ResponderCommitmentState::new(); + older.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + assert!( + older.lookup_by_hash(&legacy_pin).is_some(), + "the chunk-only commitment is still answerable" + ); + assert!( + older.lookup_by_hash(¤t_pin).is_none(), + "the one committing a pointer cannot be rebuilt without the sidecar" + ); + } + + /// A sidecar that describes other commitments, as one left by a snapshot + /// from before a rollback might, attaches nothing to these. + #[test] + fn a_sidecar_only_describes_the_commitments_it_names() { + let other = ResponderCommitmentState::new(); + other.rotate(mixed(&[1], &[10])); + let sidecar = other.snapshot().pointer_leaves_bytes().expect("sidecar"); + + let built = mixed(&[1], &[10]); + let pin = built.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(built); + let mut reloaded = + PersistedRetention::from_bytes(&state.snapshot().to_bytes().expect("bytes")) + .expect("deserialize"); + assert!(reloaded.attach_pointer_leaves(&sidecar)); + let fresh = ResponderCommitmentState::new(); + fresh.restore(&reloaded); + assert!( + fresh.lookup_by_hash(&pin).is_none(), + "another commitment's pointer leaves were applied to this one" + ); + assert!( + !reloaded.attach_pointer_leaves(&[0xff; 7]), + "garbage is refused" + ); + } + /// A corrupt snapshot blob decodes to `None`, so the caller fails open with /// empty retention rather than trusting garbage. #[test] @@ -1071,6 +1269,7 @@ mod tests { commitment: built.commitment().clone(), leaf_keys: vec![key(1)], expires_at_unix: None, + pointer_keys: Vec::new(), }], has_current: true, }; @@ -1102,6 +1301,7 @@ mod tests { commitment: built.commitment().clone(), leaf_keys: leaf_keys.clone(), expires_at_unix, + pointer_keys: Vec::new(), }], has_current: false, // not current -> retention depends on the stamp }; diff --git a/src/replication/config.rs b/src/replication/config.rs index 9150d584..276ba1bb 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -426,7 +426,14 @@ pub const REPLICATION_PROTOCOL_ID: &str = "autonomi.ant.replication.v2"; /// possession/repair/commitment-fetch) with no per-peer limiter. A truly /// zero-penalty rollout needs an upstream `send_request` that does not /// auto-report trust; tracked as a saorsa-core follow-up. -pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v1"; +/// +/// `v2` (ADR-0016): commitments carry pointer leaves, committed as +/// `(address, pointer_leaf_hash(address))`, and round 2 answers one with the +/// whole signed record. A `v1` auditor would reject a pointer leaf in round 1 +/// as a content-address mismatch and penalise an honest holder, so the two +/// versions must not audit each other: the same bounded pause as the `v1` +/// introduction, for the same reason. +pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v2"; /// 10 MiB — maximum replication wire message size (accommodates hint batches). const REPLICATION_MESSAGE_SIZE_MIB: usize = 10; @@ -1487,7 +1494,7 @@ mod tests { assert_eq!(REPLICATION_PROTOCOL_ID, "autonomi.ant.replication.v2"); assert_eq!( SUBTREE_AUDIT_PROTOCOL_ID, - "autonomi.ant.replication.subtree-audit.v1" + "autonomi.ant.replication.subtree-audit.v2" ); assert_ne!(REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID); } diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 54498cdc..73d91f45 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -25,6 +25,7 @@ pub mod config; pub mod fresh; pub mod neighbor_sync; pub mod paid_list; +pub mod pointer; pub mod possession; pub mod protocol; pub mod pruning; @@ -66,6 +67,7 @@ use crate::payment::{ PaymentVerifier, VerificationContext, MAX_PAYMENT_PROOF_SIZE_BYTES, MIN_PAYMENT_PROOF_SIZE_BYTES, }; +use crate::pointer::store::PointerStore; use crate::replication::audit::AuditTickResult; use crate::replication::audit_coordinator::AuditChallengeCoordinator; use crate::replication::audit_metrics::{ @@ -1815,6 +1817,10 @@ pub struct ReplicationEngine { /// When present, `start()` spawns a drainer task that calls /// `replicate_fresh` for each event. fresh_write_rx: Option>, + /// Pointer replication (ADR-0016), when this node stores pointers. + pointers: Option>, + /// Receiver for fresh pointer writes, taken by `start()`. + pointer_fresh_rx: Option>, /// Sender for delayed possession-check events (ADR-0003). The fresh-write /// drainer pushes the responsible close-group peers here after each fresh /// replication; the possession-check scheduler drains the paired receiver. @@ -1948,6 +1954,8 @@ impl ReplicationEngine { config.subtree_round1_max_concurrent, ), fresh_write_rx: Some(fresh_write_rx), + pointers: None, + pointer_fresh_rx: None, possession_check_tx, possession_check_rx: Some(possession_check_rx), monetized_pin_tx, @@ -2031,6 +2039,7 @@ impl ReplicationEngine { pub async fn rebuild_commitment_now(&self) -> Result<()> { rebuild_and_rotate_commitment( &self.storage, + self.pointers.as_ref().map(|p| p.store()), &self.identity, &self.commitment_state, &self.p2p_node, @@ -2205,6 +2214,36 @@ impl ReplicationEngine { }) } + /// Replicate pointers too (ADR-0016): the records in `store`, and the + /// fresh writes the pointer PUT handler sends on `fresh_writes`. + /// + /// Call before [`Self::start`]. + pub fn with_pointers( + &mut self, + store: PointerStore, + fresh_writes: mpsc::UnboundedReceiver, + ) { + self.pointers = Some(Arc::new(pointer::PointerReplication::new( + store, + Arc::clone(&self.storage), + Arc::clone(&self.p2p_node), + Arc::clone(&self.payment_verifier), + Arc::clone(&self.config), + Arc::clone(&self.is_bootstrapping), + Arc::clone(&self.send_semaphore), + self.shutdown.clone(), + self.detached_task_tracker.clone(), + ))); + self.pointer_fresh_rx = Some(fresh_writes); + } + + /// The pointer replication, when enabled. Tests use it to drive rounds. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn pointer_replication(&self) -> Option<&Arc> { + self.pointers.as_ref() + } + /// Start all background tasks. /// /// `dht_events` must be subscribed **before** `P2PNode::start()` so that @@ -2232,6 +2271,12 @@ impl ReplicationEngine { self.start_bootstrap_sync(dht_events); self.start_fresh_write_drainer(); self.start_possession_check_scheduler(); + if let Some(pointers) = &self.pointers { + self.task_handles.push(pointers.start_verification_loop()); + if let Some(writes) = self.pointer_fresh_rx.take() { + self.task_handles.push(pointers.start_fresh_drainer(writes)); + } + } // ADR-0004: deterministic first audit of commitments that backed a // payment (surfaced by the verifier cross-check). self.start_first_audit_drainer(); @@ -2870,6 +2915,7 @@ impl ReplicationEngine { paid_notify_worker_semaphore, paid_notify_admission_semaphore, paid_notify_responder_inflight, + pointers: self.pointers.clone(), shutdown: shutdown.clone(), detached_task_tracker, }; @@ -3101,6 +3147,9 @@ impl ReplicationEngine { DhtNetworkEvent::PeerRemoved { peer_id } => { sync_state.write().await.remove_peer(&peer_id); repair_proofs.write().await.remove_peer(&peer_id); + if let Some(pointers) = &handler_context.pointers { + pointers.forget_peer(&peer_id); + } update_bootstrap_after_peer_removed( &peer_id, &handler_context.bootstrap_state, @@ -3140,6 +3189,7 @@ impl ReplicationEngine { } fn start_neighbor_sync_loop(&mut self) { + let pointers = self.pointers.clone(); let p2p = Arc::clone(&self.p2p_node); let storage = Arc::clone(&self.storage); let paid_list = Arc::clone(&self.paid_list); @@ -3216,6 +3266,7 @@ impl ReplicationEngine { &sig_verify_attempts, &audit_challenge_coordinator, &gossip_audit, + pointers.as_ref(), ) => {} } } @@ -3355,6 +3406,7 @@ impl ReplicationEngine { let config = Arc::clone(&self.config); let sync_trigger = Arc::clone(&self.sync_trigger); let recent_provers = Arc::clone(&self.recent_provers); + let pointer_store = self.pointers.as_ref().map(|p| p.store().clone()); let handle = tokio::spawn(async move { // Build the first commitment immediately on startup so a @@ -3377,9 +3429,15 @@ impl ReplicationEngine { // unchanged — preserving the reloaded current pin; otherwise the // reloaded roots stay answerable as retained slots until their gossip // TTL lapses. Persistence is handled by the retention-persist loop. - if let Err(e) = - rebuild_and_rotate_commitment(&storage, &identity, &commitment_state, &p2p, &config) - .await + if let Err(e) = rebuild_and_rotate_commitment( + &storage, + pointer_store.as_ref(), + &identity, + &commitment_state, + &p2p, + &config, + ) + .await { warn!("Initial commitment build failed: {e}"); } else { @@ -3393,6 +3451,7 @@ impl ReplicationEngine { ) => { if let Err(e) = rebuild_and_rotate_commitment( &storage, + pointer_store.as_ref(), &identity, &commitment_state, &p2p, @@ -4310,6 +4369,8 @@ struct ReplicationMessageHandlerContext { paid_notify_worker_semaphore: Arc, paid_notify_admission_semaphore: Arc, paid_notify_responder_inflight: Arc>>, + /// Pointer replication, when this node stores pointers. + pointers: Option>, /// The engine's shutdown token, for detached responder work. /// /// Workers on [`Self::detached_task_tracker`] race this around their @@ -4399,6 +4460,12 @@ const fn replication_message_class(body: &ReplicationMessageBody) -> &'static st ReplicationMessageBody::SubtreeSliceResponse(_) => "subtree_slice_response", ReplicationMessageBody::GetCommitmentByPin(_) => "commitment_pin_request", ReplicationMessageBody::GetCommitmentByPinResponse(_) => "commitment_pin_response", + ReplicationMessageBody::PointerFreshOffer(_) => "pointer_fresh_offer", + ReplicationMessageBody::PointerHints(_) => "pointer_hints", + ReplicationMessageBody::PointerFetchRequest(_) => "pointer_fetch_request", + ReplicationMessageBody::PointerFetchResponse(_) => "pointer_fetch_response", + ReplicationMessageBody::PointerStateRequest(_) => "pointer_state_request", + ReplicationMessageBody::PointerStateResponse(_) => "pointer_state_response", } } @@ -5126,6 +5193,7 @@ async fn handle_replication_message( let storage = Arc::clone(&ctx.storage); let p2p_node = Arc::clone(&ctx.p2p_node); let my_commitment_state = Arc::clone(&ctx.my_commitment_state); + let pointer_store = ctx.pointers.as_ref().map(|p| p.store().clone()); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -5138,9 +5206,10 @@ async fn handle_replication_message( let storage_commitment_audit::Round1Work { response, content_bytes, - } = storage_commitment_audit::handle_subtree_challenge_measured( + } = storage_commitment_audit::handle_subtree_challenge_measured_with_pointers( &challenge, &storage, + pointer_store.as_ref(), p2p_node.peer_id(), bootstrapping, Some(&my_commitment_state), @@ -5312,6 +5381,7 @@ async fn handle_replication_message( let storage = Arc::clone(&ctx.storage); let p2p_node = Arc::clone(&ctx.p2p_node); let my_commitment_state = Arc::clone(&ctx.my_commitment_state); + let pointer_store = ctx.pointers.as_ref().map(|p| p.store().clone()); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -5320,14 +5390,16 @@ async fn handle_replication_message( let _guard = guard; // global permit + per-peer slot, held until done let worker_started = Instant::now(); let processing_started = Instant::now(); - let response = storage_commitment_audit::handle_subtree_slice_challenge( - &challenge, - &storage, - p2p_node.peer_id(), - bootstrapping, - Some(&my_commitment_state), - ) - .await; + let response = + storage_commitment_audit::handle_subtree_slice_challenge_with_pointers( + &challenge, + &storage, + pointer_store.as_ref(), + p2p_node.peer_id(), + bootstrapping, + Some(&my_commitment_state), + ) + .await; let processing = processing_started.elapsed(); let response_kind = subtree_slice_response_kind(&response); let response_send_started = Instant::now(); @@ -5461,6 +5533,42 @@ async fn handle_replication_message( drop(guard); Ok(()) } + // Pointers (ADR-0016). A node that does not store pointers ignores + // them, and so is never marked capable and never asked anything. + ReplicationMessageBody::PointerFreshOffer(offer) => { + if let Some(pointers) = &ctx.pointers { + pointers.accept_offer_detached(*source, offer); + } + Ok(()) + } + ReplicationMessageBody::PointerHints(hints) => { + if let Some(pointers) = &ctx.pointers { + pointers.handle_hints(*source, hints.hints).await; + } + Ok(()) + } + ReplicationMessageBody::PointerFetchRequest(request) => { + if let Some(pointers) = &ctx.pointers { + pointers.serve_fetch_detached( + *source, + request, + msg.request_id, + rr_message_id.map(ToOwned::to_owned), + ); + } + Ok(()) + } + ReplicationMessageBody::PointerStateRequest(request) => { + if let Some(pointers) = &ctx.pointers { + pointers.serve_state_detached( + *source, + request, + msg.request_id, + rr_message_id.map(ToOwned::to_owned), + ); + } + Ok(()) + } // Response messages are handled by their respective request initiators. ReplicationMessageBody::FreshReplicationResponse(_) | ReplicationMessageBody::NeighborSyncResponse(_) @@ -5469,7 +5577,9 @@ async fn handle_replication_message( | ReplicationMessageBody::AuditResponse(_) | ReplicationMessageBody::SubtreeAuditResponse(_) | ReplicationMessageBody::SubtreeSliceResponse(_) - | ReplicationMessageBody::GetCommitmentByPinResponse(_) => Ok(()), + | ReplicationMessageBody::GetCommitmentByPinResponse(_) + | ReplicationMessageBody::PointerFetchResponse(_) + | ReplicationMessageBody::PointerStateResponse(_) => Ok(()), } } @@ -6426,6 +6536,12 @@ async fn dispatch_neighbor_sync_request( received_at: Instant, rr_message_id: Option<&str>, ) -> Result<()> { + // A peer syncing with us gets our pointer hints as well — including a + // node that is bootstrapping, which is how it learns the pointers it + // should hold. + if let Some(pointers) = &ctx.pointers { + pointers.push_hints_detached(vec![source]); + } let guard = match admit_bounded_responder( &ctx.neighbor_sync_responder_admission_semaphore, &ctx.neighbor_sync_responder_inflight, @@ -7390,6 +7506,7 @@ async fn run_neighbor_sync_round( sig_verify_attempts: &Arc>>, audit_challenge_coordinator: &Arc, gossip_audit: &GossipAuditTrigger, + pointers: Option<&Arc>, ) { let self_id = *p2p_node.peer_id(); let bootstrapping = *is_bootstrapping.read().await; @@ -7431,6 +7548,11 @@ async fn run_neighbor_sync_round( audit_challenge_coordinator, }) .await; + if let Some(pointers) = pointers { + pointers + .prune_pass(allow_remote_prune_audits, Some(commitment_state)) + .await; + } // Take fresh close-neighbor snapshot (DHT query, no lock held). let neighbors = @@ -7470,6 +7592,9 @@ async fn run_neighbor_sync_round( } debug!("Neighbor sync: syncing with {} peers", batch.len()); + if let Some(pointers) = pointers { + pointers.push_hints_detached(batch.clone()); + } // Snapshot our current commitment once per round so all peers in // this batch see the same thing (gossip is the responder's attestation; @@ -9871,7 +9996,24 @@ async fn load_commitment_retention(state: &ResponderCommitmentState, path: &Path return; } }; - if let Some(persisted) = PersistedRetention::from_bytes(&bytes) { + if let Some(mut persisted) = PersistedRetention::from_bytes(&bytes) { + let sidecar = pointer_leaves_path(path); + match tokio::fs::read(&sidecar).await { + Ok(leaves) => { + if !persisted.attach_pointer_leaves(&leaves) { + warn!( + "Commitment retention: corrupt pointer leaves at {}; \ + commitments holding pointers will not be restored", + sidecar.display() + ); + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!( + "Commitment retention: failed to read {}: {e}", + sidecar.display() + ), + } state.restore(&persisted); info!( "Commitment retention: reloaded {} slot(s) from {}", @@ -9891,23 +10033,38 @@ async fn load_commitment_retention(state: &ResponderCommitmentState, path: &Path /// needless disk writes on idle nodes. On success updates `last` to the bytes /// written; on a serialization/write error the existing on-disk snapshot is left /// intact (never truncated). +/// +/// The pointer leaves go to a sidecar written first (ADR-0016), so the +/// retention file never names a commitment whose pointer leaves are not yet +/// on disk. A crash between the two leaves a sidecar ahead of the file, which +/// is harmless: it is keyed by commitment hash. async fn persist_retention_if_changed( state: &ResponderCommitmentState, path: &Path, last: &mut Option>, ) { - let Some(bytes) = state.snapshot().to_bytes() else { + let snapshot = state.snapshot(); + let (Some(bytes), Some(leaves)) = (snapshot.to_bytes(), snapshot.pointer_leaves_bytes()) else { warn!("Commitment retention: serialization failed; keeping previous snapshot"); return; }; - if last.as_deref() == Some(bytes.as_slice()) { + let mut combined = leaves.clone(); + combined.extend_from_slice(&bytes); + if last.as_deref() == Some(combined.as_slice()) { return; } - if write_retention_atomic(path, bytes.clone()).await { - *last = Some(bytes); + if write_retention_atomic(&pointer_leaves_path(path), leaves).await + && write_retention_atomic(path, bytes).await + { + *last = Some(combined); } } +/// Where the pointer leaves of the retention at `path` are kept. +fn pointer_leaves_path(path: &Path) -> PathBuf { + path.with_file_name("commitment_retention_pointers.bin") +} + /// Durably write `bytes` to `path`: temp file → fsync temp → atomic rename → /// fsync parent dir (so the rename itself survives a crash). Returns `true` on /// success. Only the retention-persist loop writes this path, so a fixed temp @@ -9952,11 +10109,16 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { /// BLAKE3(content)`, so `bytes_hash := key` and we don't have to /// re-read each chunk's bytes to compute the leaf hash. /// +/// The pointers this node is responsible for are committed alongside, each as +/// `(address, pointer_leaf_hash(address))` (ADR-0016), so they are audited and +/// priced exactly as chunks are. +/// /// Skips (returns `Ok(())`) if the key set is empty — no commitment to /// rotate. The auditor side handles "no commitment for this peer" by /// falling back to the legacy plain-digest audit path. async fn rebuild_and_rotate_commitment( storage: &Arc, + pointers: Option<&PointerStore>, identity: &Arc, state: &Arc, p2p: &Arc, @@ -9989,8 +10151,16 @@ async fn rebuild_and_rotate_commitment( keys.push(k); } } + // The same rule for pointers: an out-of-range pointer leaves the next + // commitment, and the pruner reclaims it once no retained slot holds it. + let mut pointer_keys = Vec::new(); + for state in pointers.map(PointerStore::held_states).unwrap_or_default() { + if admission::is_responsible(&self_id, &state.address, p2p, config.close_group_size).await { + pointer_keys.push(state.address); + } + } - if keys.is_empty() { + if keys.is_empty() && pointer_keys.is_empty() { // There used to be a second branch here that dropped every retained root outright // when the node looked empty. It is gone, and the reason is worth keeping. // @@ -10038,12 +10208,11 @@ async fn rebuild_and_rotate_commitment( // to more than the protocol limit; auditor would reject the // commitment otherwise). let cap = commitment::MAX_COMMITMENT_KEY_COUNT as usize; - if keys.len() > cap { + let total = keys.len().saturating_add(pointer_keys.len()); + if total > cap { warn!( - "Commitment rotation: key set ({}) exceeds MAX_COMMITMENT_KEY_COUNT ({}); \ - truncating — investigate as this likely means a misconfiguration", - keys.len(), - cap + "Commitment rotation: key set ({total}) exceeds MAX_COMMITMENT_KEY_COUNT ({cap}); \ + truncating — investigate as this likely means a misconfiguration" ); } @@ -10065,7 +10234,28 @@ async fn rebuild_and_rotate_commitment( // earn credit for `key`. If this module is ever reused for // non-content-addressed records, that `(k, k)` shortcut AND the verifier // gate must be replaced with `(key, BLAKE3(bytes))` computed from real bytes. - let entries: Vec<_> = keys.into_iter().take(cap).map(|k| (k, k)).collect(); + // + // Pointers are that case, and are handled by being told apart rather than + // hashed: a pointer leaf is `(address, pointer_leaf_hash(address))`, which + // the verifier accepts only for a leaf of exactly a pointer's size, and + // round 2 then demands the whole signed record in place of a Bao slice. + // The root binds which pointers are held, never their current state, so an + // update does not move it. + // + // A pointer address that is also a chunk key would need a BLAKE3 preimage; + // the dedup only keeps such a collision from failing the whole build. + let mut entries: Vec<_> = keys + .into_iter() + .map(|k| (k, k)) + .chain( + pointer_keys + .into_iter() + .map(|address| (address, commitment::pointer_leaf_hash(&address))), + ) + .collect(); + entries.sort_by_key(|(k, _)| *k); + entries.dedup_by_key(|(k, _)| *k); + entries.truncate(cap); // No-op-rotation guard: compute just the Merkle root from `entries` // and compare against the currently-advertised commitment's root. @@ -10120,8 +10310,9 @@ async fn rebuild_and_rotate_commitment( let hash = hex::encode(built.hash()); let key_count = built.commitment().key_count; + let pointer_count = built.tree().pointer_count(); state.rotate(built); - info!("Storage commitment rotated: hash={hash} key_count={key_count}"); + info!("Storage commitment rotated: hash={hash} key_count={key_count} pointers={pointer_count}"); // Counted only on the paths where the advertised commitment now genuinely reflects // the committable set, never merely on having read it. The retirement gate is what // consumes this, and it authorises deleting the legacy store. diff --git a/src/replication/pointer.rs b/src/replication/pointer.rs new file mode 100644 index 00000000..df62a0f4 --- /dev/null +++ b/src/replication/pointer.rs @@ -0,0 +1,1304 @@ +//! Pointer replication (ADR-0016). +//! +//! Pointers ride the same replication machinery chunks do — the same close +//! groups, the same neighbour-sync rounds and churn triggers, the same quorum, +//! pruning and possession rules — but not the chunk pipeline itself. That +//! pipeline assumes a record never changes and that its key is the hash of its +//! bytes. A pointer's address is stable while its state changes, and two honest +//! replicas may hold different valid signatures over one state. So a pointer is +//! replicated by *state*: +//! +//! - **Fresh.** A node that accepts a paid state from a client forwards the +//! record, with the proof that paid for it, to the rest of the close group. +//! Each receiver checks the signature, its own responsibility and the payment +//! itself before storing it. +//! - **Repair.** Every neighbour-sync round pushes hints: the states the sender +//! holds that the receiver should hold. A receiver that lacks a hinted state, +//! or holds an older one, asks the close group which state each holds, adopts +//! the best state a quorum of them hold exactly, and fetches it from one of +//! them. The record verifies itself; the quorum stands in for the payment +//! proof, as presence quorum does for a chunk. +//! - **Pruning.** A record this node has been out of range of for the +//! hysteresis period is deleted — at once if the node is far outside the +//! group, otherwise only once all but one of the current close group prove +//! they hold that state or a newer one by returning a valid record. +//! - **Possession.** Some minutes after offering a fresh state, the node asks +//! each close-group member for the record. A member that is still responsible +//! and cannot produce that state or a newer one is penalised. +//! +//! Requests go only to peers that have sent a pointer message themselves (see +//! [`PointerReplication::is_capable`]). A peer built before pointers cannot +//! decode them, and saorsa-core counts an unanswered request against the peer +//! it was sent to; asking only capable peers keeps older peers out of it. Every +//! neighbour-sync round pushes hints, empty or not, so capability is learned +//! within a cycle. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use ant_protocol::pointer::{Pointer, PointerState, POINTER_WIRE_LEN}; +use futures::stream::{self, StreamExt}; +use parking_lot::Mutex; +use rand::Rng; +use saorsa_core::identity::PeerId; +use saorsa_core::{P2PNode, TrustEvent}; +use tokio::sync::{mpsc, RwLock, Semaphore}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use crate::ant_protocol::XorName; +use crate::logging::{debug, info, warn}; +use crate::payment::{ + PaymentVerifier, VerificationContext, MAX_PAYMENT_PROOF_SIZE_BYTES, + MIN_PAYMENT_PROOF_SIZE_BYTES, +}; +use crate::pointer::store::{Inspected, PointerStore}; +use crate::replication::admission; +use crate::replication::commitment_state::ResponderCommitmentState; +use crate::replication::config::{ + storage_admission_width, ReplicationConfig, FRESH_REPLICATION_DELIVERY_MAX_RETRIES, + REPLICATION_PROTOCOL_ID, +}; +use crate::replication::protocol::{ + PointerFetchRequest, PointerFetchResponse, PointerFreshOffer, PointerHints, + PointerStateRequest, PointerStateResponse, PointerStateSummary, ReplicationMessage, + ReplicationMessageBody, MAX_POINTER_HINTS_PER_MESSAGE, MAX_POINTER_STATE_REQUEST_ADDRESSES, +}; +use crate::replication::pruning::prune_proofs_needed; +use crate::storage::{CapacityVerdict, ChunkStore}; + +use super::REPLICATION_TRUST_WEIGHT; + +/// Inbound fresh offers verified at once. An offer costs a signature check and +/// an on-chain payment lookup, so a flood is dropped rather than queued; one +/// that is dropped is repaired by the next neighbour-sync round. +const MAX_CONCURRENT_OFFERS: usize = 16; + +/// Fetch and state requests served at once. +const MAX_CONCURRENT_SERVES: usize = 32; + +/// How often the pending hints are looked at. +const VERIFICATION_TICK: Duration = Duration::from_millis(500); + +/// Most addresses awaiting verification at once, so a hint flood cannot grow +/// memory without bound. A hint dropped here comes back next round. +const MAX_PENDING: usize = 65_536; + +/// Most addresses verified per tick. +const MAX_VERIFICATIONS_PER_TICK: usize = 256; + +/// Verifications and fetches in flight at once within a tick. +const VERIFICATION_CONCURRENCY: usize = 8; + +/// Undecided rounds an address gets before it is dropped. It comes back with +/// the next hint. +const MAX_UNDECIDED_ROUNDS: u32 = 5; + +/// Wait before retrying an undecided address, doubled each round. +const UNDECIDED_BACKOFF: Duration = Duration::from_secs(30); + +/// Most prune candidates examined per pass. +const MAX_PRUNE_CANDIDATES_PER_PASS: usize = 256; + +/// One peer's answer about one address: the state it holds there, if any. +type StateAnswer = ((PeerId, XorName), Option); + +/// A pointer state this node accepted from a paying client, to be offered to +/// the rest of its close group. +pub struct PointerFreshWrite { + /// The record, in its canonical encoding. + pub record: Vec, + /// The proof that paid for its state. + pub payment_proof: Vec, +} + +/// A state this node was told about and has not yet verified. +#[derive(Debug, Clone)] +struct Pending { + /// The best state hinted so far. + wanted: PointerState, + /// Undecided rounds so far. + attempts: u32, + /// Not to be looked at again before this. + not_before: Instant, +} + +/// What a verification round decided for one address. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Verdict { + /// A quorum of the close group hold this state and it beats what is held + /// here: fetch it from one of `holders`. + Adopt { + /// The state to fetch. + state: PointerState, + /// The peers that said they hold exactly it. + holders: Vec, + }, + /// Nothing reached quorum, but the peers that did not answer could still + /// make one. Ask again later. + Undecided, + /// No state that beats what is held can reach quorum. + Refused, +} + +/// Decide what a round of answers supports. +/// +/// `group_size` is the whole close group this node would ask, capable or not: +/// a peer that cannot be asked counts as unanswered, not as a vote. The quorum +/// is therefore the one a chunk would need, and a network where few peers +/// understand pointers yet cannot repair on the word of those few. +pub(crate) fn evaluate( + held: Option<&PointerState>, + group_size: usize, + answers: &[(PeerId, Option)], + quorum_needed: usize, +) -> Verdict { + if group_size == 0 || quorum_needed == 0 { + return Verdict::Undecided; + } + let mut by_state: Vec<(PointerState, Vec)> = Vec::new(); + for (peer, answer) in answers { + let Some(state) = answer else { continue }; + match by_state + .iter_mut() + .find(|(known, _)| known.state_id == state.state_id) + { + Some((_, holders)) => holders.push(*peer), + None => by_state.push((*state, vec![*peer])), + } + } + let beats_held = |state: &PointerState| held.is_none_or(|held| state.replaces(held)); + + let best = by_state + .iter() + .filter(|(state, holders)| holders.len() >= quorum_needed && beats_held(state)) + .fold( + None::<&(PointerState, Vec)>, + |best, candidate| match best { + Some(current) if !candidate.0.replaces(¤t.0) => Some(current), + _ => Some(candidate), + }, + ); + if let Some((state, holders)) = best { + return Verdict::Adopt { + state: *state, + holders: holders.clone(), + }; + } + + let unanswered = group_size.saturating_sub(answers.len()); + let largest = by_state + .iter() + .filter(|(state, _)| beats_held(state)) + .map(|(_, holders)| holders.len()) + .max() + .unwrap_or(0); + if largest.saturating_add(unanswered) >= quorum_needed { + Verdict::Undecided + } else { + Verdict::Refused + } +} + +/// Pointer replication for one node. See the module documentation. +pub struct PointerReplication { + store: PointerStore, + chunks: Arc, + p2p: Arc, + payments: Arc, + config: Arc, + is_bootstrapping: Arc>, + /// Outbound record transfers, shared with chunk replication. + send_semaphore: Arc, + offer_permits: Arc, + serve_permits: Arc, + /// Peers that have sent a pointer message: the only ones asked anything. + capable: Mutex>, + /// Hinted states awaiting verification, by address. + pending: Mutex>, + /// When each held address was first seen continuously out of range. + out_of_range: Mutex>, + shutdown: CancellationToken, + tracker: TaskTracker, +} + +impl PointerReplication { + /// Pointer replication over `store`, sharing the engine's resources. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + store: PointerStore, + chunks: Arc, + p2p: Arc, + payments: Arc, + config: Arc, + is_bootstrapping: Arc>, + send_semaphore: Arc, + shutdown: CancellationToken, + tracker: TaskTracker, + ) -> Self { + Self { + store, + chunks, + p2p, + payments, + config, + is_bootstrapping, + send_semaphore, + offer_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OFFERS)), + serve_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_SERVES)), + capable: Mutex::new(HashSet::new()), + pending: Mutex::new(HashMap::new()), + out_of_range: Mutex::new(HashMap::new()), + shutdown, + tracker, + } + } + + /// The pointer store this replication serves from. + pub(crate) fn store(&self) -> &PointerStore { + &self.store + } + + // ----------------------------------------------------------------------- + // Capability + // ----------------------------------------------------------------------- + + fn mark_capable(&self, peer: &PeerId) { + self.capable.lock().insert(*peer); + } + + /// Whether `peer` has sent a pointer message, and may therefore be asked. + pub fn is_capable(&self, peer: &PeerId) -> bool { + self.capable.lock().contains(peer) + } + + /// Forget a peer that left the routing table. + pub(crate) fn forget_peer(&self, peer: &PeerId) { + self.capable.lock().remove(peer); + } + + /// Addresses currently awaiting verification. Tests only. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn pending_len(&self) -> usize { + self.pending.lock().len() + } + + // ----------------------------------------------------------------------- + // Messaging + // ----------------------------------------------------------------------- + + async fn send_one_way(&self, peer: &PeerId, body: ReplicationMessageBody) -> bool { + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body, + }; + let Ok(bytes) = msg.encode() else { + warn!("Failed to encode a pointer replication message"); + return false; + }; + self.p2p + .send_message(peer, REPLICATION_PROTOCOL_ID, bytes, &[]) + .await + .is_ok() + } + + async fn request( + &self, + peer: &PeerId, + body: ReplicationMessageBody, + timeout: Duration, + ) -> Option { + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body, + }; + let bytes = msg.encode().ok()?; + let response = self + .p2p + .send_request(peer, REPLICATION_PROTOCOL_ID, bytes, timeout) + .await + .ok()?; + ReplicationMessage::decode(&response.data) + .ok() + .map(|msg| msg.body) + } + + async fn penalise(&self, peer: &PeerId) { + self.p2p + .report_trust_event( + peer, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + + /// Ask `peer` for the record it holds at `address`, and check what comes + /// back: a valid signature and the right address. Anything else counts + /// against the peer. + async fn fetch_record(&self, peer: &PeerId, address: &XorName) -> Option { + let body = self + .request( + peer, + ReplicationMessageBody::PointerFetchRequest(PointerFetchRequest { + address: *address, + }), + self.config.fetch_request_timeout, + ) + .await?; + let ReplicationMessageBody::PointerFetchResponse(response) = body else { + return None; + }; + if response.address != *address { + return None; + } + let bytes = response.record?; + match Pointer::from_bytes(&bytes) { + Ok(record) if record.address() == *address => Some(record), + Ok(_) | Err(_) => { + debug!( + "Peer {peer} served an invalid pointer record for {}", + hex::encode(address) + ); + self.penalise(peer).await; + None + } + } + } + + // ----------------------------------------------------------------------- + // Hints + // ----------------------------------------------------------------------- + + /// Push hints to `peers` in the background: for each, the states this + /// node holds whose close group, in this node's view, includes that peer. + /// + /// Every peer gets at least one message, empty or not. That is how a peer + /// learns this node understands pointers. + pub(crate) fn push_hints_detached(self: &Arc, peers: Vec) { + if peers.is_empty() { + return; + } + let this = Arc::clone(self); + self.tracker + .spawn(async move { this.push_hints(&peers).await }); + } + + /// Push hints to `peers` and wait until they are sent. This is what each + /// neighbour-sync round runs in the background; tests call it to drive a + /// round. + pub async fn push_hints(&self, peers: &[PeerId]) { + let self_id = *self.p2p.peer_id(); + let mut by_peer: HashMap> = + peers.iter().map(|peer| (*peer, Vec::new())).collect(); + for state in self.store.held_states() { + if self.shutdown.is_cancelled() { + return; + } + let group = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&state.address, self.config.close_group_size) + .await; + for node in group { + if node.peer_id == self_id { + continue; + } + if let Some(hints) = by_peer.get_mut(&node.peer_id) { + hints.push(state.into()); + } + } + } + for (peer, hints) in by_peer { + let batches: Vec> = if hints.is_empty() { + vec![Vec::new()] + } else { + hints + .chunks(MAX_POINTER_HINTS_PER_MESSAGE) + .map(<[PointerStateSummary]>::to_vec) + .collect() + }; + for hints in batches { + self.send_one_way( + &peer, + ReplicationMessageBody::PointerHints(PointerHints { hints }), + ) + .await; + } + } + } + + /// Take in hints from `source`: queue each hinted state this node should + /// hold and lacks, or holds an older state than. + pub(crate) async fn handle_hints(&self, source: PeerId, hints: Vec) { + self.mark_capable(&source); + if hints.is_empty() { + return; + } + // As for chunks: only a peer in our own routing table may tell us what + // to hold. + if !self.p2p.dht_manager().is_in_routing_table(&source).await { + debug!("Dropping pointer hints from {source}: not in the routing table"); + return; + } + let self_id = *self.p2p.peer_id(); + let width = storage_admission_width(self.config.close_group_size); + for summary in hints.into_iter().take(MAX_POINTER_HINTS_PER_MESSAGE) { + let hinted = PointerState::from(summary); + match self.store.state(&hinted.address) { + Some(held) => { + if held.state_id == hinted.state_id || !hinted.replaces(&held) { + continue; + } + } + None => { + if !admission::is_responsible(&self_id, &hinted.address, &self.p2p, width).await + { + continue; + } + } + } + self.enqueue(hinted); + } + } + + fn enqueue(&self, hinted: PointerState) { + let mut pending = self.pending.lock(); + let len = pending.len(); + match pending.get_mut(&hinted.address) { + Some(entry) => { + if hinted.replaces(&entry.wanted) { + entry.wanted = hinted; + entry.attempts = 0; + entry.not_before = Instant::now(); + } + } + None if len < MAX_PENDING => { + pending.insert( + hinted.address, + Pending { + wanted: hinted, + attempts: 0, + not_before: Instant::now(), + }, + ); + } + None => {} + } + } + + // ----------------------------------------------------------------------- + // Verification and fetch + // ----------------------------------------------------------------------- + + /// Start the loop that verifies and fetches hinted states. + pub(crate) fn start_verification_loop(self: &Arc) -> JoinHandle<()> { + let this = Arc::clone(self); + tokio::spawn(async move { + loop { + tokio::select! { + () = this.shutdown.cancelled() => break, + () = tokio::time::sleep(VERIFICATION_TICK) => this.verify_due().await, + } + } + }) + } + + /// Verify and fetch whatever is due now. Also the tests' way to drive it. + pub async fn verify_due(&self) { + // As for chunks (ADR-0011): a node that cannot store what it would + // fetch does not spend the network's time finding it. The hints wait, + // and come back each round if they are dropped meanwhile. + if self.chunks.capacity_verdict() == CapacityVerdict::Full { + return; + } + let now = Instant::now(); + let due: Vec<(XorName, PointerState)> = self + .pending + .lock() + .iter() + .filter(|(_, entry)| entry.not_before <= now) + .take(MAX_VERIFICATIONS_PER_TICK) + .map(|(address, entry)| (*address, entry.wanted)) + .collect(); + if due.is_empty() { + return; + } + + let self_id = *self.p2p.peer_id(); + let mut groups: HashMap> = HashMap::new(); + let mut asked: HashMap> = HashMap::new(); + for (address, _) in &due { + let group: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local(address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .filter(|peer| *peer != self_id) + .collect(); + for peer in group.iter().filter(|peer| self.is_capable(peer)) { + asked.entry(*peer).or_default().push(*address); + } + groups.insert(*address, group); + } + + let answers = self.ask_states(asked).await; + + let decisions: Vec<(XorName, Verdict)> = due + .iter() + .map(|(address, _)| { + let group = groups.get(address).map_or(&[][..], Vec::as_slice); + let answered: Vec<(PeerId, Option)> = group + .iter() + .filter_map(|peer| { + answers + .get(&(*peer, *address)) + .map(|answer| (*peer, *answer)) + }) + .collect(); + let held = self.store.state(address); + let verdict = evaluate( + held.as_ref(), + group.len(), + &answered, + self.config.quorum_needed(group.len()), + ); + (*address, verdict) + }) + .collect(); + + let outcomes: Vec<(XorName, bool)> = stream::iter(decisions) + .map(|(address, verdict)| async move { + match verdict { + Verdict::Adopt { state, holders } => { + let stored = self.fetch_and_store(address, state, &holders).await; + (address, stored) + } + Verdict::Undecided => (address, false), + Verdict::Refused => { + info!( + "No quorum backs a newer state for pointer {}", + hex::encode(address) + ); + self.pending.lock().remove(&address); + (address, true) + } + } + }) + .buffer_unordered(VERIFICATION_CONCURRENCY) + .collect() + .await; + + let now = Instant::now(); + let mut pending = self.pending.lock(); + for (address, settled) in outcomes { + if settled { + pending.remove(&address); + continue; + } + if let Some(entry) = pending.get_mut(&address) { + entry.attempts = entry.attempts.saturating_add(1); + if entry.attempts >= MAX_UNDECIDED_ROUNDS { + pending.remove(&address); + } else { + let backoff = UNDECIDED_BACKOFF + .saturating_mul(2u32.saturating_pow(entry.attempts.saturating_sub(1))); + entry.not_before = now + backoff; + } + } + } + } + + /// Ask each peer which state it holds at the addresses listed for it. + async fn ask_states( + &self, + asked: HashMap>, + ) -> HashMap<(PeerId, XorName), Option> { + let requests: Vec<(PeerId, Vec)> = asked + .into_iter() + .flat_map(|(peer, addresses)| { + addresses + .chunks(MAX_POINTER_STATE_REQUEST_ADDRESSES) + .map(|chunk| (peer, chunk.to_vec())) + .collect::>() + }) + .collect(); + let replies: Vec> = stream::iter(requests) + .map(|(peer, addresses)| async move { + let body = self + .request( + &peer, + ReplicationMessageBody::PointerStateRequest(PointerStateRequest { + addresses: addresses.clone(), + }), + self.config.verification_request_timeout, + ) + .await; + let Some(ReplicationMessageBody::PointerStateResponse(response)) = body else { + return Vec::new(); + }; + addresses + .iter() + .zip(response.states) + .map(|(address, summary)| { + // An answer about a different address is no answer. + let state = summary + .filter(|summary| summary.address == *address) + .map(PointerState::from); + ((peer, *address), state) + }) + .collect() + }) + .buffer_unordered(VERIFICATION_CONCURRENCY) + .collect() + .await; + replies.into_iter().flatten().collect() + } + + /// Fetch `wanted` from one of `holders` and store it. Whether it is now + /// held, or nothing further can come of this round. + async fn fetch_and_store( + &self, + address: XorName, + wanted: PointerState, + holders: &[PeerId], + ) -> bool { + for holder in holders { + let Some(record) = self.fetch_record(holder, &address).await else { + continue; + }; + // The quorum backed this state. A holder that has moved on since + // serves a different one, which that quorum says nothing about. + if record.state_id() != wanted.state_id { + continue; + } + match self.store_verified(record, None).await { + Ok(_) => return true, + Err(e) => { + warn!( + "Could not store replicated pointer {}: {e}", + hex::encode(address) + ); + return false; + } + } + } + false + } + + /// Store a record whose signature has been checked, if it still belongs + /// here and beats what is held. + async fn store_verified( + &self, + record: Pointer, + width: Option, + ) -> crate::error::Result { + let state = record.state(); + let self_id = *self.p2p.peer_id(); + let width = width.unwrap_or_else(|| storage_admission_width(self.config.close_group_size)); + let held = self.store.state(&state.address).is_some(); + if !held && !admission::is_responsible(&self_id, &state.address, &self.p2p, width).await { + return Ok(false); + } + if !self.store.admits(&state) { + return Ok(false); + } + let reservation = self.chunks.reserve(POINTER_WIRE_LEN as u64)?; + self.store.commit(record, Some(reservation)).await?; + Ok(true) + } + + // ----------------------------------------------------------------------- + // Serving + // ----------------------------------------------------------------------- + + /// Answer a fetch request in the background. + pub(crate) fn serve_fetch_detached( + self: &Arc, + source: PeerId, + request: PointerFetchRequest, + request_id: u64, + rr_message_id: Option, + ) { + self.mark_capable(&source); + let this = Arc::clone(self); + self.tracker.spawn(async move { + let Ok(_permit) = this.serve_permits.acquire().await else { + return; + }; + // `get` verifies the signature before serving, so a record damaged + // on this disk is never handed on. + let record = match this.store.get(&request.address).await { + Ok(Some(record)) => Some(record.to_bytes()), + Ok(None) | Err(_) => None, + }; + super::send_replication_response( + &source, + &this.p2p, + request_id, + ReplicationMessageBody::PointerFetchResponse(PointerFetchResponse { + address: request.address, + record, + }), + rr_message_id.as_deref(), + ) + .await; + }); + } + + /// Answer a state request in the background, from the index. + pub(crate) fn serve_state_detached( + self: &Arc, + source: PeerId, + request: PointerStateRequest, + request_id: u64, + rr_message_id: Option, + ) { + self.mark_capable(&source); + let this = Arc::clone(self); + self.tracker.spawn(async move { + let Ok(_permit) = this.serve_permits.acquire().await else { + return; + }; + let states = request + .addresses + .iter() + .take(MAX_POINTER_STATE_REQUEST_ADDRESSES) + .map(|address| this.store.state(address).map(PointerStateSummary::from)) + .collect(); + super::send_replication_response( + &source, + &this.p2p, + request_id, + ReplicationMessageBody::PointerStateResponse(PointerStateResponse { states }), + rr_message_id.as_deref(), + ) + .await; + }); + } + + // ----------------------------------------------------------------------- + // Fresh replication + // ----------------------------------------------------------------------- + + /// Start the loop that offers freshly paid states to their close groups. + pub(crate) fn start_fresh_drainer( + self: &Arc, + mut writes: mpsc::UnboundedReceiver, + ) -> JoinHandle<()> { + let this = Arc::clone(self); + tokio::spawn(async move { + loop { + tokio::select! { + () = this.shutdown.cancelled() => break, + write = writes.recv() => match write { + Some(write) => this.replicate_fresh(write).await, + None => break, + }, + } + } + }) + } + + /// Offer a freshly paid state to the rest of its close group, and schedule + /// the check that they took it. + pub async fn replicate_fresh(self: &Arc, write: PointerFreshWrite) { + let Ok(state) = PointerState::parse(&write.record) else { + warn!("A fresh pointer write did not parse; not replicating it"); + return; + }; + let self_id = *self.p2p.peer_id(); + let targets: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&state.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .filter(|peer| *peer != self_id) + .collect(); + + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body: ReplicationMessageBody::PointerFreshOffer(PointerFreshOffer { + record: write.record, + proof_of_payment: write.payment_proof, + }), + }; + let Ok(encoded) = msg.encode() else { + warn!( + "Failed to encode a fresh pointer offer for {}", + hex::encode(state.address) + ); + return; + }; + let encoded = Arc::new(encoded); + for peer in &targets { + let p2p = Arc::clone(&self.p2p); + let bytes = Arc::clone(&encoded); + let semaphore = Arc::clone(&self.send_semaphore); + let peer = *peer; + self.tracker.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + return; + }; + for attempt in 0..=FRESH_REPLICATION_DELIVERY_MAX_RETRIES { + match p2p + .send_message(&peer, REPLICATION_PROTOCOL_ID, bytes.as_ref().clone(), &[]) + .await + { + Ok(()) => break, + Err(e) => debug!( + "Fresh pointer offer to {peer} failed (attempt {}): {e}", + attempt + 1 + ), + } + } + }); + } + debug!( + "Fresh pointer {} offered to {} peers", + hex::encode(state.address), + targets.len() + ); + self.schedule_possession_check(state, targets); + } + + /// Take a fresh offer from `source` in the background, unless too many are + /// already being verified. + pub(crate) fn accept_offer_detached( + self: &Arc, + source: PeerId, + offer: PointerFreshOffer, + ) { + self.mark_capable(&source); + let Ok(permit) = Arc::clone(&self.offer_permits).try_acquire_owned() else { + info!("Dropping a fresh pointer offer from {source}: too many in flight"); + return; + }; + let this = Arc::clone(self); + self.tracker.spawn(async move { + let _permit = permit; + this.accept_offer(source, offer).await; + }); + } + + /// Verify and store a fresh offer. Cheapest checks first; the signature and + /// the on-chain payment lookup only for a state that would change something + /// here and that this node is responsible for. + async fn accept_offer(&self, source: PeerId, offer: PointerFreshOffer) { + if offer.record.len() != POINTER_WIRE_LEN + || !(MIN_PAYMENT_PROOF_SIZE_BYTES..=MAX_PAYMENT_PROOF_SIZE_BYTES) + .contains(&offer.proof_of_payment.len()) + { + debug!("Dropping a malformed fresh pointer offer from {source}"); + self.penalise(&source).await; + return; + } + let parsed = match self.store.inspect(&offer.record).await { + Ok(Inspected::Candidate(parsed)) => parsed, + Ok(Inspected::Unchanged(_) | Inspected::Stale(_)) => return, + Err(e) => { + debug!("Dropping an unparseable fresh pointer offer from {source}: {e}"); + self.penalise(&source).await; + return; + } + }; + let state = *parsed.state(); + let self_id = *self.p2p.peer_id(); + // As for a chunk: a fresh offer is taken across the wider paid width. + if !admission::is_responsible( + &self_id, + &state.address, + &self.p2p, + self.config.paid_list_close_group_size, + ) + .await + { + return; + } + if !self.store.admits(&state) + || self + .chunks + .check_capacity_for(POINTER_WIRE_LEN as u64) + .is_err() + { + return; + } + let record = match self.store.verify(parsed).await { + Ok(record) => record, + Err(e) => { + debug!("Fresh pointer offer from {source} does not verify: {e}"); + self.penalise(&source).await; + return; + } + }; + if let Err(e) = self + .payments + .verify_pointer_payment_in( + &state.address, + &state.state_id, + &offer.proof_of_payment, + VerificationContext::FreshReplication, + ) + .await + { + info!( + "Fresh pointer offer for {} from {source} is not paid for: {e}", + hex::encode(state.address) + ); + return; + } + match self + .store_verified(record, Some(self.config.paid_list_close_group_size)) + .await + { + Ok(true) => debug!( + "Stored fresh pointer {} from {source}", + hex::encode(state.address) + ), + Ok(false) => {} + Err(e) => warn!( + "Could not store fresh pointer {}: {e}", + hex::encode(state.address) + ), + } + } + + // ----------------------------------------------------------------------- + // Possession + // ----------------------------------------------------------------------- + + fn schedule_possession_check(self: &Arc, fresh: PointerState, peers: Vec) { + if peers.is_empty() { + return; + } + let min = self.config.possession_check_delay_min; + let max = self.config.possession_check_delay_max.max(min); + let delay = if max > min { + rand::thread_rng().gen_range(min..=max) + } else { + min + }; + let this = Arc::clone(self); + self.tracker.spawn(async move { + tokio::select! { + () = this.shutdown.cancelled() => return, + () = tokio::time::sleep(delay) => {} + } + this.check_possession(fresh, &peers).await; + }); + } + + /// Ask each peer for the record, and penalise any still responsible for it + /// that cannot produce `fresh` or a state that replaces it. + pub async fn check_possession(&self, fresh: PointerState, peers: &[PeerId]) { + let self_id = *self.p2p.peer_id(); + let group: HashSet = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&fresh.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + for peer in peers { + // A peer that has left the group owes nothing, and one that cannot + // be asked is never judged on its silence. + if *peer == self_id || !group.contains(peer) || !self.is_capable(peer) { + continue; + } + let holds = self + .fetch_record(peer, &fresh.address) + .await + .is_some_and(|record| { + let state = record.state(); + state.state_id == fresh.state_id || state.replaces(&fresh) + }); + if !holds { + warn!( + "Peer {peer} does not hold pointer {} it was offered", + hex::encode(fresh.address) + ); + self.penalise(peer).await; + } + } + } + + // ----------------------------------------------------------------------- + // Pruning + // ----------------------------------------------------------------------- + + /// Delete records this node has been out of range of for the hysteresis + /// period, where that is safe. Run when a neighbour-sync cycle completes. + /// + /// `allow_remote` is false while bootstrapping: candidacy is still tracked, + /// but nothing that needs other peers' proof is decided. + /// + /// A record a retained storage commitment still commits to is never + /// deleted, exactly as for a chunk: an auditor pinning that commitment may + /// still open it, and a node that deleted it would fail that audit. + pub async fn prune_pass( + &self, + allow_remote: bool, + commitment_state: Option<&ResponderCommitmentState>, + ) { + let committed = |address: &XorName| commitment_state.is_some_and(|cs| cs.is_held(address)); + let self_id = *self.p2p.peer_id(); + let retention = storage_admission_width(self.config.close_group_size); + let now = Instant::now(); + let held = self.store.held_states(); + + let mut candidates = Vec::new(); + for state in &held { + if admission::is_responsible(&self_id, &state.address, &self.p2p, retention).await { + self.out_of_range.lock().remove(&state.address); + continue; + } + let first_seen = *self.out_of_range.lock().entry(state.address).or_insert(now); + if now.duration_since(first_seen) >= self.config.prune_hysteresis_duration { + candidates.push(*state); + } + } + { + let held: HashSet = held.iter().map(|state| state.address).collect(); + self.out_of_range + .lock() + .retain(|address, _| held.contains(address)); + } + + let mut deleted = 0usize; + for state in candidates.into_iter().take(MAX_PRUNE_CANDIDATES_PER_PASS) { + if self.shutdown.is_cancelled() { + return; + } + if committed(&state.address) { + continue; + } + let wide = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self( + &state.address, + self.config.paid_list_close_group_size, + ) + .await; + let far = wide.len() >= self.config.paid_list_close_group_size + && !wide.iter().any(|node| node.peer_id == self_id); + let confirmed = if far { + true + } else if allow_remote && !*self.is_bootstrapping.read().await { + self.others_hold(&state).await + } else { + false + }; + // Revalidate just before deleting: the group may have moved back. + if confirmed + && !committed(&state.address) + && !admission::is_responsible(&self_id, &state.address, &self.p2p, retention).await + && self.delete(&state.address).await + { + deleted += 1; + } + } + if deleted > 0 { + info!("Pruned {deleted} pointer records this node is no longer responsible for"); + } + } + + /// Whether all but one of the current close group prove they hold `ours` + /// or a newer state, by returning a valid record. + async fn others_hold(&self, ours: &PointerState) -> bool { + let group: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local(&ours.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + let needed = prune_proofs_needed(group.len()); + if needed == 0 { + return false; + } + let mut proofs = 0usize; + for peer in group.iter().filter(|peer| self.is_capable(peer)) { + let proves = self + .fetch_record(peer, &ours.address) + .await + .is_some_and(|record| { + let state = record.state(); + state.state_id == ours.state_id || state.replaces(ours) + }); + if proves { + proofs += 1; + if proofs >= needed { + return true; + } + } + } + false + } + + async fn delete(&self, address: &XorName) -> bool { + self.out_of_range.lock().remove(address); + match self.store.delete(address).await { + Ok(true) => { + self.chunks.release(POINTER_WIRE_LEN as u64); + true + } + Ok(false) => false, + Err(e) => { + warn!("Could not prune pointer {}: {e}", hex::encode(address)); + false + } + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind}; + + fn state(counter: u64, target: u8, id: u8) -> PointerState { + PointerState { + state_id: [id; 32], + address: [7; 32], + counter, + target: PointerTarget::new(PointerTargetKind::Chunk, [target; 32]), + } + } + + fn peer(byte: u8) -> PeerId { + PeerId::from_bytes([byte; 32]) + } + + fn answers(list: &[(u8, Option)]) -> Vec<(PeerId, Option)> { + list.iter().map(|(p, s)| (peer(*p), *s)).collect() + } + + #[test] + fn a_state_a_quorum_holds_is_adopted_from_its_holders() { + let newer = state(3, 1, 30); + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(newer)), + (2, Some(newer)), + (3, Some(newer)), + (4, Some(newer)), + (5, None), + ]), + 4, + ); + match got { + Verdict::Adopt { state, holders } => { + assert_eq!(state.state_id, newer.state_id); + assert_eq!(holders.len(), 4); + } + other => panic!("expected Adopt, got {other:?}"), + } + } + + #[test] + fn a_state_below_quorum_is_never_adopted() { + // One peer, or three, holding a state is not the network's word. + let lone = state(9, 1, 90); + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(lone)), + (2, Some(lone)), + (3, Some(lone)), + (4, None), + (5, None), + (6, None), + (7, None), + ]), + 4, + ); + assert_eq!(got, Verdict::Refused); + + // With one peer silent, three holders plus that peer could still make + // four: undecided, not refused. + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(lone)), + (2, Some(lone)), + (3, Some(lone)), + (4, None), + (5, None), + (6, None), + ]), + 4, + ); + assert_eq!(got, Verdict::Undecided); + } + + #[test] + fn silence_is_not_a_vote_so_an_unanswered_group_stays_undecided() { + let newer = state(3, 1, 30); + let got = evaluate(None, 7, &answers(&[(1, Some(newer)), (2, Some(newer))]), 4); + assert_eq!(got, Verdict::Undecided); + } + + #[test] + fn nothing_older_than_what_is_held_is_adopted() { + let held = state(5, 1, 50); + let older = state(4, 1, 40); + let got = evaluate( + Some(&held), + 7, + &answers(&[ + (1, Some(older)), + (2, Some(older)), + (3, Some(older)), + (4, Some(older)), + (5, Some(older)), + ]), + 4, + ); + assert_eq!(got, Verdict::Refused); + } + + #[test] + fn of_two_states_with_quorum_the_merge_winner_is_adopted() { + let lower = state(2, 9, 20); + let winner = state(2, 1, 21); + assert!(winner.replaces(&lower)); + let got = evaluate( + None, + 8, + &answers(&[ + (1, Some(lower)), + (2, Some(lower)), + (3, Some(lower)), + (4, Some(lower)), + (5, Some(winner)), + (6, Some(winner)), + (7, Some(winner)), + (8, Some(winner)), + ]), + 4, + ); + match got { + Verdict::Adopt { state, .. } => assert_eq!(state.state_id, winner.state_id), + other => panic!("expected the winner, got {other:?}"), + } + } + + #[test] + fn a_group_nobody_can_be_asked_in_is_undecided() { + assert_eq!(evaluate(None, 0, &[], 0), Verdict::Undecided); + } +} diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index 3ef7d0fa..b2ca7962 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -217,6 +217,25 @@ pub enum ReplicationMessageBody { GetCommitmentByPin(GetCommitmentByPin), /// Response to [`Self::GetCommitmentByPin`]. GetCommitmentByPinResponse(GetCommitmentByPinResponse), + + // === Pointers (ADR-0016) === + // APPENDED for the same reason: every earlier discriminant keeps its value. + // A peer built before pointers cannot decode these and drops them. The two + // one-way pushes are then simply lost, and the requests are only ever sent + // to peers that have sent a pointer message themselves, so an older peer is + // never asked something it cannot answer and never penalised for silence. + /// A newly paid pointer state, with the proof that paid for it. + PointerFreshOffer(PointerFreshOffer), + /// The pointer states the sender holds that the receiver should hold too. + PointerHints(PointerHints), + /// Ask a peer for the pointer record it holds at an address. + PointerFetchRequest(PointerFetchRequest), + /// Response to [`Self::PointerFetchRequest`]. + PointerFetchResponse(PointerFetchResponse), + /// Ask a peer which state it holds at each of some addresses. + PointerStateRequest(PointerStateRequest), + /// Response to [`Self::PointerStateRequest`]. + PointerStateResponse(PointerStateResponse), } // --------------------------------------------------------------------------- @@ -235,7 +254,7 @@ pub enum ReplicationMessageBody { // modules that do not carry any shared engine handle. /// Number of [`ReplicationMessageBody`] variants (the counter-table width). -pub(crate) const N_REPLICATION_VARIANTS: usize = 17; +pub(crate) const N_REPLICATION_VARIANTS: usize = 23; static REPL_TX_BYTES: [AtomicU64; N_REPLICATION_VARIANTS] = [const { AtomicU64::new(0) }; N_REPLICATION_VARIANTS]; @@ -270,6 +289,12 @@ impl ReplicationMessageBody { Self::SubtreeSliceResponse(_) => 14, Self::GetCommitmentByPin(_) => 15, Self::GetCommitmentByPinResponse(_) => 16, + Self::PointerFreshOffer(_) => 17, + Self::PointerHints(_) => 18, + Self::PointerFetchRequest(_) => 19, + Self::PointerFetchResponse(_) => 20, + Self::PointerStateRequest(_) => 21, + Self::PointerStateResponse(_) => 22, } } @@ -336,7 +361,7 @@ impl BodyFamily { pub(crate) fn family_of_variant(index: usize) -> Option { match index { 11..=14 => Some(BodyFamily::SubtreeAudit), - 0..=10 | 15 | 16 => Some(BodyFamily::Core), + 0..=10 | 15..=22 => Some(BodyFamily::Core), _ => None, } } @@ -984,6 +1009,117 @@ pub enum GetCommitmentByPinResponse { }, } +// --------------------------------------------------------------------------- +// Pointer Messages (ADR-0016) +// --------------------------------------------------------------------------- + +/// Most hints one [`PointerHints`] message carries. A sender with more splits +/// them across messages; a receiver ignores anything past this in one message. +pub const MAX_POINTER_HINTS_PER_MESSAGE: usize = 8192; + +/// Most addresses one [`PointerStateRequest`] asks about. A responder answers +/// only this many, and the requester treats the rest as unanswered. +pub const MAX_POINTER_STATE_REQUEST_ADDRESSES: usize = 512; + +/// A pointer state, as a hint or as an answer about what a node holds. +/// +/// The fields the merge rule orders by, and the state they identify, so a +/// receiver can tell whether a hinted state would replace what it holds without +/// fetching it. Unauthenticated on its own: nothing is stored on the strength +/// of a summary. A receiver acts on one only through quorum verification and a +/// fetched record whose signature it checks itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PointerStateSummary { + /// The pointer's address. + pub address: XorName, + /// The state's identifier. + pub state_id: XorName, + /// The update counter, the merge rule's first key. + pub counter: u64, + /// The target's kind tag. + pub target_tag: u8, + /// The target's address. With the tag, the merge rule's second key. + pub target_address: XorName, +} + +impl From for PointerStateSummary { + fn from(state: ant_protocol::pointer::PointerState) -> Self { + Self { + address: state.address, + state_id: state.state_id, + counter: state.counter, + target_tag: state.target.kind_tag(), + target_address: state.target.address, + } + } +} + +impl From for ant_protocol::pointer::PointerState { + fn from(summary: PointerStateSummary) -> Self { + Self { + state_id: summary.state_id, + address: summary.address, + counter: summary.counter, + target: ant_protocol::pointer::PointerTarget::from_raw_tag( + summary.target_tag, + summary.target_address, + ), + } + } +} + +/// A newly paid pointer state, with the proof that paid for it. +/// +/// Sent by a node that accepted the state from a paying client to the rest of +/// the pointer's close group, which verifies the proof itself. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerFreshOffer { + /// The pointer record, in its canonical encoding. + pub record: Vec, + /// Serialized proof of payment for the record's state. + pub proof_of_payment: Vec, +} + +/// The pointer states the sender holds that the receiver should hold too. +/// +/// Sent on every neighbour-sync round, empty if there is nothing to say: an +/// empty message still tells the receiver the sender understands pointers. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerHints { + /// The states, at most [`MAX_POINTER_HINTS_PER_MESSAGE`] of them. + pub hints: Vec, +} + +/// Ask a peer for the pointer record it holds at an address. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct PointerFetchRequest { + /// The pointer's address. + pub address: XorName, +} + +/// Response to [`PointerFetchRequest`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerFetchResponse { + /// Echo of the requested address. + pub address: XorName, + /// The record held there, if any. + pub record: Option>, +} + +/// Ask a peer which state it holds at each of some addresses. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerStateRequest { + /// The addresses, at most [`MAX_POINTER_STATE_REQUEST_ADDRESSES`]. + pub addresses: Vec, +} + +/// Response to [`PointerStateRequest`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerStateResponse { + /// One entry per requested address, in order: the state held, or `None`. + pub states: Vec>, +} + // --------------------------------------------------------------------------- // Audit Messages // --------------------------------------------------------------------------- @@ -1230,8 +1366,26 @@ pub enum SubtreeSliceItem { /// The committed key the responder could not serve. key: XorName, }, + /// The responder holds this committed pointer (ADR-0016) and serves the + /// whole signed record. A pointer's address is not a content hash, so no + /// slice of it could be authenticated against the address; the record can + /// be, by its signature, and against the nonced root round 1 bound over + /// its bytes. One per requested pointer key, whatever blocks were named. + PointerRecord { + /// The requested key: the pointer's address. + key: XorName, + /// The record held now, and the one an update replaced since round 1 + /// if there was one, each in its canonical encoding. At most + /// [`MAX_POINTER_RECORDS_PER_ITEM`]: round 1 bound one of them, and + /// the responder cannot tell which without keeping round 1's answer. + records: Vec>, + }, } +/// Most records one [`SubtreeSliceItem::PointerRecord`] may carry: the one +/// held now and the one it replaced. +pub const MAX_POINTER_RECORDS_PER_ITEM: usize = 2; + /// Response to a [`SubtreeSliceChallenge`] (round 2). /// /// The contract is **coalesced and order-independent**: the responder groups the @@ -1572,6 +1726,18 @@ mod tests { ReplicationMessageBody::GetCommitmentByPinResponse( GetCommitmentByPinResponse::NotRetained { pin: z }, ), + ReplicationMessageBody::PointerFreshOffer(PointerFreshOffer { + record: vec![], + proof_of_payment: vec![], + }), + ReplicationMessageBody::PointerHints(PointerHints { hints: vec![] }), + ReplicationMessageBody::PointerFetchRequest(PointerFetchRequest { address: z }), + ReplicationMessageBody::PointerFetchResponse(PointerFetchResponse { + address: z, + record: None, + }), + ReplicationMessageBody::PointerStateRequest(PointerStateRequest { addresses: vec![] }), + ReplicationMessageBody::PointerStateResponse(PointerStateResponse { states: vec![] }), ] } @@ -1769,7 +1935,7 @@ mod tests { fn an_undeclared_variant_index_classifies_as_nothing() { let declared = all_bodies().len(); assert_eq!( - declared, 17, + declared, 23, "update this test's bounds when a variant is added or removed" ); for index in [declared, declared + 1, 99, usize::MAX] { diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index f99a4e70..c20b8e60 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -17,23 +17,27 @@ use crate::logging::{debug, info, warn}; use rand::Rng; use crate::ant_protocol::XorName; -use crate::replication::commitment::{commitment_hash, StorageCommitment}; +use crate::pointer::store::{PointerStore, SUPERSEDED_RETENTION}; +use crate::replication::commitment::{commitment_hash, pointer_leaf_hash, StorageCommitment}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ ReplicationConfig, MAX_SLICE_OPENINGS, SUBTREE_AUDIT_PROTOCOL_ID, - SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES, + SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES, SUBTREE_SESSION_TTL, }; use crate::replication::protocol::{ RejectKind, ReplicationMessage, ReplicationMessageBody, SubtreeAuditChallenge, SubtreeAuditResponse, SubtreeSliceChallenge, SubtreeSliceItem, SubtreeSliceOpening, - SubtreeSliceResponse, + SubtreeSliceResponse, MAX_POINTER_RECORDS_PER_ITEM, }; use crate::replication::recent_provers::RecentProvers; +use crate::replication::slice::nonced_block_root; use crate::replication::subtree::{ - select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, + select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeLeaf, + SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; use crate::storage::ChunkStore; +use ant_protocol::pointer::{Pointer, POINTER_WIRE_LEN}; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -723,12 +727,76 @@ pub(crate) fn evaluate_subtree_structure( // for honest content-addressed data the two are identical, so this never fails // an honest holder, and it re-binds Chain 1's `bytes_hash` check to the credited // `key`. - if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { + // + // A pointer (ADR-0016) is the one other honest leaf shape: its address is not + // a content hash, so it is committed as `(key, pointer_leaf_hash(key))` at the + // fixed record length, and round 2 proves possession with the whole signed + // record instead of a slice. Anything else is still rejected. + if proof + .leaves + .iter() + .any(|l| l.bytes_hash != l.key && !is_pointer_leaf(l)) + { return Err(AuditFailureReason::DigestMismatch); } Ok(()) } +// Round 2 may be owed the record an update replaced after round 1 bound it. +const _: () = assert!( + SUPERSEDED_RETENTION.as_secs() > SUBTREE_SESSION_TTL.as_secs(), + "a replaced pointer record must outlive the audit session that may be owed it" +); + +/// Whether a round-1 leaf commits a pointer (ADR-0016): committed under +/// [`pointer_leaf_hash`] of its key, at the fixed record length. +fn is_pointer_leaf(leaf: &SubtreeLeaf) -> bool { + leaf.bytes_hash == pointer_leaf_hash(&leaf.key) + && usize::try_from(leaf.content_len).ok() == Some(POINTER_WIRE_LEN) +} + +/// Check the round-2 item for a pointer leaf: one of the records served must +/// prove it. An admitted absence is `KeyAbsent`; anything else that does not +/// prove it, including no item at all, is `DigestMismatch`. +fn verify_pointer_item( + nonce: &[u8; 32], + challenged_peer_bytes: &[u8; 32], + leaf: &SubtreeLeaf, + items: &[SubtreeSliceItem], +) -> Result<(), AuditFailureReason> { + let served = items.iter().find_map(|it| match it { + SubtreeSliceItem::PointerRecord { key, records } if key == &leaf.key => { + Some(Some(records.as_slice())) + } + SubtreeSliceItem::Absent { key } if key == &leaf.key => Some(None), + _ => None, + }); + match served { + Some(Some(records)) + if records + .iter() + .any(|record| proves_pointer(nonce, challenged_peer_bytes, leaf, record)) => + { + Ok(()) + } + Some(None) => Err(AuditFailureReason::KeyAbsent), + _ => Err(AuditFailureReason::DigestMismatch), + } +} + +/// Whether `record` proves the pointer `leaf` commits: its signature verifies, +/// it belongs at the leaf's address, and it is the record round 1 bound the +/// leaf's nonced root over. +fn proves_pointer( + nonce: &[u8; 32], + challenged_peer_bytes: &[u8; 32], + leaf: &SubtreeLeaf, + record: &[u8], +) -> bool { + nonced_block_root(nonce, challenged_peer_bytes, &leaf.key, record) == leaf.nonced_root + && Pointer::from_bytes(record).is_ok_and(|pointer| pointer.address() == leaf.key) +} + /// The auditor's **freshly-randomised** spot-check sample of the round-1 proof: /// `count` distinct leaves (deduplicated, in increasing-index order) whose /// original bytes the auditor will demand in round 2. @@ -870,6 +938,7 @@ pub(crate) fn verify_slice_response( let requested_keys: HashSet = openings.iter().map(|(leaf, _)| leaf.key).collect(); let mut present: HashSet<(XorName, u32)> = HashSet::new(); let mut absent: HashSet = HashSet::new(); + let mut records: HashSet = HashSet::new(); for it in items { let ok = match it { SubtreeSliceItem::Present { @@ -878,11 +947,23 @@ pub(crate) fn verify_slice_response( requested_blocks.contains(&(*key, *block_index)) && present.insert((*key, *block_index)) && !absent.contains(key) + && !records.contains(key) } SubtreeSliceItem::Absent { key } => { requested_keys.contains(key) && absent.insert(*key) && !present.iter().any(|(k, _)| k == key) + && !records.contains(key) + } + SubtreeSliceItem::PointerRecord { + key, + records: served, + } => { + requested_keys.contains(key) + && (1..=MAX_POINTER_RECORDS_PER_ITEM).contains(&served.len()) + && records.insert(*key) + && !absent.contains(key) + && !present.iter().any(|(k, _)| k == key) } }; if !ok { @@ -892,6 +973,18 @@ pub(crate) fn verify_slice_response( let mut checked = 0usize; for (leaf, block_index) in openings { + // A pointer leaf is proved by the whole signed record: it must verify, + // belong at the committed address, and be the record round 1 bound its + // nonced root over, which the responder had to read before it knew what + // would be sampled. An update between the rounds does not fail an + // honest holder: it serves the record it held then beside the new one. + if is_pointer_leaf(leaf) { + if let Err(reason) = verify_pointer_item(nonce, challenged_peer_bytes, leaf, items) { + return AuditVerdict::Fail(reason); + } + checked += 1; + continue; + } let block_index = *block_index; // Match the responder's item for exactly this (key, block_index). A // missing item, an explicit Absent, or a different block is a provable lie. @@ -1020,7 +1113,13 @@ async fn verify_subtree_response( .iter() .flat_map(|leaf| { let leaf = (*leaf).clone(); - block_indices_for_leaf(leaf.content_len) + // A pointer is served whole, so it takes one opening, at block 0. + let indices = if is_pointer_leaf(&leaf) { + vec![0] + } else { + block_indices_for_leaf(leaf.content_len) + }; + indices .into_iter() .map(move |block_index| (leaf.clone(), block_index)) }) @@ -1271,6 +1370,27 @@ pub async fn handle_subtree_challenge_measured( self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, +) -> Round1Work { + handle_subtree_challenge_measured_with_pointers( + challenge, + storage, + None, + self_peer_id, + is_bootstrapping, + commitment_state, + ) + .await +} + +/// [`handle_subtree_challenge_measured`] for a node that also commits pointers +/// (ADR-0016): committed pointer leaves are answered from `pointers`. +pub async fn handle_subtree_challenge_measured_with_pointers( + challenge: &SubtreeAuditChallenge, + storage: &ChunkStore, + pointers: Option<&PointerStore>, + self_peer_id: &PeerId, + is_bootstrapping: bool, + commitment_state: Option<&Arc>, ) -> Round1Work { // The accumulator is threaded in rather than returned per-arm so that every // exit reports its work by construction: a new early return cannot forget to @@ -1279,6 +1399,7 @@ pub async fn handle_subtree_challenge_measured( let response = subtree_challenge_response( challenge, storage, + pointers, self_peer_id, is_bootstrapping, commitment_state, @@ -1298,6 +1419,7 @@ pub async fn handle_subtree_challenge_measured( async fn subtree_challenge_response( challenge: &SubtreeAuditChallenge, storage: &ChunkStore, + pointers: Option<&PointerStore>, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1361,7 +1483,62 @@ async fn subtree_challenge_response( // Read chunk bytes one leaf at a time so peak memory is bounded regardless // of subtree size, hashing each into its plain + nonced leaf. let mut leaves = Vec::with_capacity(plan.leaf_keys.len()); - for key in &plan.leaf_keys { + for (position, key) in plan.leaf_keys.iter().enumerate() { + // A pointer leaf (ADR-0016) commits no bytes in the tree, since its + // bytes change with every update. Round 1 binds them here instead: the + // nonced root over the record held now, which round 2 must reproduce. + // So a node answers from records it holds, not ones it could fetch once + // it learns which few are sampled. A loss is admitted exactly as a + // missing chunk is, and a read error is transient as for a chunk. + if plan.leaf_is_pointer.get(position).copied().unwrap_or(false) { + *content_bytes = content_bytes.saturating_add(SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES); + let key_hex = hex::encode(key); + let read = match pointers { + Some(store) => store.record_bytes(key).await, + None => Ok(None), + }; + let record = match read { + Ok(Some(record)) => record, + Ok(None) => { + warn!("Subtree audit: committed pointer {key_hex} is not held"); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Protocol, + reason: format!("missing bytes for committed key: {key_hex}"), + }; + } + Err(e) => { + warn!( + "Subtree audit: read error for committed pointer {key_hex}: {e} \ + (rejecting as transient, not a confirmed failure)" + ); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Transient, + reason: format!("transient storage read error: {e}"), + }; + } + }; + // Top up to the record read, as for a chunk. + let content = i64::try_from(record.len()).unwrap_or(i64::MAX); + *content_bytes = content_bytes.saturating_add( + content + .saturating_sub(SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES) + .max(0), + ); + leaves.push(SubtreeLeaf { + key: *key, + bytes_hash: pointer_leaf_hash(key), + content_len: u32::try_from(POINTER_WIRE_LEN).unwrap_or(u32::MAX), + nonced_root: nonced_block_root( + &challenge.nonce, + &challenge.challenged_peer_id, + key, + &record, + ), + }); + continue; + } // Charge the fixed cost of ATTEMPTING a leaf before the read, because // it is owed whether or not the read succeeds: the LMDB lookup and its // retries, and the blocking-task round trip below. Charging only @@ -1551,6 +1728,28 @@ pub async fn handle_subtree_slice_challenge( self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, +) -> SubtreeSliceResponse { + handle_subtree_slice_challenge_with_pointers( + challenge, + storage, + None, + self_peer_id, + is_bootstrapping, + commitment_state, + ) + .await +} + +/// [`handle_subtree_slice_challenge`] for a node that also commits pointers +/// (ADR-0016): a committed pointer is answered with its whole signed record. +#[allow(clippy::too_many_lines)] +pub async fn handle_subtree_slice_challenge_with_pointers( + challenge: &SubtreeSliceChallenge, + storage: &ChunkStore, + pointers: Option<&PointerStore>, + self_peer_id: &PeerId, + is_bootstrapping: bool, + commitment_state: Option<&Arc>, ) -> SubtreeSliceResponse { if is_bootstrapping { return SubtreeSliceResponse::Bootstrapping { @@ -1682,6 +1881,33 @@ pub async fn handle_subtree_slice_challenge( let mut items = Vec::with_capacity(challenge.openings.len()); for key in key_order { let indices = indices_by_key.remove(&key).unwrap_or_default(); + if built.tree().commits_pointer(&key) { + // The bytes held now, exactly as round 1 read them, and the record + // an update replaced since, if any: round 1 bound one of the two. + // The auditor verifies whichever it checks, so nothing is verified + // here. + let Some(store) = pointers else { + items.push(SubtreeSliceItem::Absent { key }); + continue; + }; + let current = match store.record_bytes(&key).await { + Ok(current) => current, + Err(e) => { + return SubtreeSliceResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Transient, + reason: format!("pointer read error: {e}"), + } + } + }; + let records: Vec> = current.into_iter().chain(store.superseded(&key)).collect(); + if records.is_empty() { + items.push(SubtreeSliceItem::Absent { key }); + } else { + items.push(SubtreeSliceItem::PointerRecord { key, records }); + } + continue; + } match serve_committed_key_openings(challenge, storage, key, indices).await { KeyServe::Items(mut built_items) => items.append(&mut built_items), KeyServe::Absent => items.push(SubtreeSliceItem::Absent { key }), @@ -2514,3 +2740,440 @@ mod tests { }; } } + +/// Pointers in the storage audit (ADR-0016), driven through the live responders +/// against real stores and judged by the auditor's own checks, so a pass here +/// is a pass on the network. +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod pointer_audit_tests { + use super::*; + use crate::replication::commitment::MerkleTree; + use crate::replication::commitment_state::BuiltCommitment; + use crate::storage::ChunkStoreConfig; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind}; + use saorsa_pqc::api::sig::ml_dsa_65; + use tempfile::TempDir; + + const CHALLENGE_ID: u64 = 7; + + /// A one-block chunk, distinct per `i`. + fn chunk(i: u8) -> Vec { + (0..=255u8).cycle().take(1024).map(|b| b ^ i).collect() + } + + /// `owner`'s pointer at `counter`. + fn pointer(owner: u8, counter: u64) -> Pointer { + let (pk, sk) = ml_dsa_65().generate_keypair_from_seed(&[owner; 32]); + let target = PointerTarget::new(PointerTargetKind::Chunk, [owner; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + /// A responder that holds and has committed to `chunks` chunks and + /// `pointers` pointers. + struct Responder { + storage: ChunkStore, + pointers: PointerStore, + state: Arc, + peer: PeerId, + peer_bytes: [u8; 32], + _dirs: (TempDir, TempDir), + } + + impl Responder { + async fn new(chunks: u8, pointers: u8) -> Self { + let chunk_dir = TempDir::new().expect("temp dir"); + let storage = ChunkStore::new(ChunkStoreConfig { + root_dir: chunk_dir.path().to_path_buf(), + ..ChunkStoreConfig::test_default() + }) + .await + .expect("chunk store"); + let pointer_dir = TempDir::new().expect("temp dir"); + let pointer_store = PointerStore::new(pointer_dir.path()) + .await + .expect("pointer store"); + + let mut entries = Vec::new(); + for i in 0..chunks { + let content = chunk(i); + let address = ChunkStore::compute_address(&content); + storage.put(&address, &content).await.expect("put chunk"); + entries.push((address, address)); + } + for owner in 0..pointers { + let record = pointer(owner, 1); + pointer_store + .put_bytes(&record.to_bytes()) + .await + .expect("put pointer"); + entries.push((record.address(), pointer_leaf_hash(&record.address()))); + } + + let (pk, sk) = ml_dsa_65().generate_keypair().expect("keypair"); + let peer_bytes = *blake3::hash(&pk.to_bytes()).as_bytes(); + let built = + BuiltCommitment::build(entries, &peer_bytes, &sk, &pk.to_bytes()).expect("build"); + let state = Arc::new(ResponderCommitmentState::new()); + state.rotate(built); + Self { + storage, + pointers: pointer_store, + state, + peer: PeerId::from_bytes(peer_bytes), + peer_bytes, + _dirs: (chunk_dir, pointer_dir), + } + } + + fn committed(&self) -> Arc { + self.state.current().expect("a current commitment") + } + + async fn round1(&self, nonce: [u8; 32]) -> SubtreeAuditResponse { + let challenge = SubtreeAuditChallenge { + challenge_id: CHALLENGE_ID, + nonce, + challenged_peer_id: self.peer_bytes, + expected_commitment_hash: self.committed().hash(), + }; + handle_subtree_challenge_measured_with_pointers( + &challenge, + &self.storage, + Some(&self.pointers), + &self.peer, + false, + Some(&self.state), + ) + .await + .response + } + + async fn round2( + &self, + nonce: [u8; 32], + openings: &[(SubtreeLeaf, u32)], + ) -> Vec { + let challenge = SubtreeSliceChallenge { + challenge_id: CHALLENGE_ID, + nonce, + challenged_peer_id: self.peer_bytes, + expected_commitment_hash: self.committed().hash(), + openings: openings + .iter() + .map(|(leaf, block_index)| SubtreeSliceOpening { + key: leaf.key, + block_index: *block_index, + }) + .collect(), + }; + match handle_subtree_slice_challenge_with_pointers( + &challenge, + &self.storage, + Some(&self.pointers), + &self.peer, + false, + Some(&self.state), + ) + .await + { + SubtreeSliceResponse::Items { items, .. } => items, + other => panic!("expected items, got {other:?}"), + } + } + + /// Round 1 as the auditor sees it: the proof, checked against the pin. + async fn proved_leaves(&self, nonce: [u8; 32]) -> Vec { + let committed = self.committed(); + match self.round1(nonce).await { + SubtreeAuditResponse::Proof { + commitment, proof, .. + } => { + assert_eq!( + evaluate_subtree_structure( + &commitment, + &proof, + &nonce, + &committed.hash(), + &self.peer_bytes, + ), + Ok(()), + "the auditor must accept the round-1 proof" + ); + proof.leaves + } + other => panic!("expected a proof, got {other:?}"), + } + } + } + + /// A nonce whose audited subtree holds both a pointer and a chunk. + fn mixed_nonce(tree: &MerkleTree) -> [u8; 32] { + (0..=255u8) + .map(|b| [b; 32]) + .find(|nonce| { + subtree_plan(tree, nonce).is_ok_and(|plan| { + plan.leaf_is_pointer.contains(&true) && plan.leaf_is_pointer.contains(&false) + }) + }) + .expect("some nonce audits a mixed subtree") + } + + /// What the auditor opens: every pointer leaf, then chunks, up to the + /// spot-check cap, with the block indices production draws. + fn openings(leaves: &[SubtreeLeaf]) -> Vec<(SubtreeLeaf, u32)> { + let (pointers, chunks): (Vec<_>, Vec<_>) = + leaves.iter().partition(|leaf| is_pointer_leaf(leaf)); + pointers + .into_iter() + .take(3) + .chain(chunks) + .take(BYTE_SPOTCHECK_MAX as usize) + .flat_map(|leaf| { + let indices = if is_pointer_leaf(leaf) { + vec![0] + } else { + block_indices_for_leaf(leaf.content_len) + }; + indices.into_iter().map(|i| (leaf.clone(), i)) + }) + .collect() + } + + fn first_pointer(openings: &[(SubtreeLeaf, u32)]) -> XorName { + openings + .iter() + .map(|(leaf, _)| leaf) + .find(|leaf| is_pointer_leaf(leaf)) + .map(|leaf| leaf.key) + .expect("a pointer is opened") + } + + #[tokio::test] + async fn a_committed_pointer_is_proved_by_its_signed_record() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let leaves = responder.proved_leaves(nonce).await; + let openings = openings(&leaves); + let items = responder.round2(nonce, &openings).await; + + assert!( + items + .iter() + .any(|item| matches!(item, SubtreeSliceItem::PointerRecord { .. })), + "a pointer is proved by its record" + ); + assert!( + items + .iter() + .any(|item| matches!(item, SubtreeSliceItem::Present { .. })), + "and the chunks beside it by their slices" + ); + assert!( + matches!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Pass { .. } + ), + "the auditor must pass an honest holder of both" + ); + } + + /// The commitment binds which pointers are held, not their state, so an + /// owner updating a pointer mid-audit cannot fail the node holding it. + #[tokio::test] + async fn an_update_between_the_rounds_does_not_fail_an_honest_holder() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let updated = first_pointer(&openings); + + let owner = (0..24u8) + .find(|owner| pointer(*owner, 1).address() == updated) + .expect("the opened pointer is one of ours"); + responder + .pointers + .put_bytes(&pointer(owner, 2).to_bytes()) + .await + .expect("update"); + + let items = responder.round2(nonce, &openings).await; + assert!(matches!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Pass { .. } + )); + } + + #[tokio::test] + async fn a_node_that_lost_a_committed_pointer_fails_round_one() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let plan = subtree_plan(responder.committed().tree(), &nonce).expect("plan"); + let lost = plan + .leaf_keys + .iter() + .zip(&plan.leaf_is_pointer) + .find_map(|(key, is_pointer)| is_pointer.then_some(*key)) + .expect("a pointer in the subtree"); + assert!(responder.pointers.delete(&lost).await.expect("delete")); + + match responder.round1(nonce).await { + SubtreeAuditResponse::Rejected { kind, .. } => { + assert_eq!( + grade_reject(kind), + RejectGrade::Confirmed, + "a lost pointer is a confirmed failure, as a lost chunk is" + ); + } + other => panic!("expected a rejection, got {other:?}"), + } + } + + #[tokio::test] + async fn a_pointer_lost_after_round_one_is_admitted_absent() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + assert!(responder + .pointers + .delete(&first_pointer(&openings)) + .await + .expect("delete")); + + let items = responder.round2(nonce, &openings).await; + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::KeyAbsent) + ); + } + + /// A node cannot prove it holds one pointer with another, genuinely signed + /// record: the record must belong at the committed address. + #[tokio::test] + async fn another_pointer_is_not_proof_of_the_one_committed() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + + let mut items = responder.round2(nonce, &openings).await; + replace_records(&mut items, &target, &[pointer(200, 1).to_bytes()]); + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + fn replace_records(items: &mut [SubtreeSliceItem], target: &XorName, with: &[Vec]) { + for item in items { + if let SubtreeSliceItem::PointerRecord { key, records } = item { + if key == target { + *records = with.to_vec(); + } + } + } + } + + /// A node that holds no pointers cannot pass by fetching the few sampled + /// records once round 2 names them: round 1 had to bind each record's + /// bytes under the nonce before anything was sampled, and it had none. + #[tokio::test] + async fn a_relay_that_fetches_records_only_in_round_two_fails() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let mut leaves = responder.proved_leaves(nonce).await; + // What a relay can say in round 1 without the bytes. + for leaf in leaves.iter_mut().filter(|leaf| is_pointer_leaf(leaf)) { + leaf.nonced_root = [0u8; 32]; + } + let openings = openings(&leaves); + // And in round 2 it serves the genuine records, fetched on demand. + let items = responder.round2(nonce, &openings).await; + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + /// Nor can it pass with another replica's copy of the very state it + /// committed: every signature is randomised, so each replica's bytes, and + /// the nonced root over them, are its own. + #[tokio::test] + async fn another_replicas_copy_of_the_same_state_is_not_proof() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + let owner = (0..24u8) + .find(|owner| pointer(*owner, 1).address() == target) + .expect("the opened pointer is one of ours"); + let replica = pointer(owner, 1); + assert_eq!( + replica.state_id(), + responder.pointers.state(&target).expect("held").state_id, + "the same state" + ); + + let mut items = responder.round2(nonce, &openings).await; + replace_records(&mut items, &target, &[replica.to_bytes()]); + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + /// A pointer item may carry the record held now and the one it replaced, + /// never more. + #[tokio::test] + async fn a_pointer_item_with_more_than_two_records_is_malformed() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + let mut items = responder.round2(nonce, &openings).await; + let held = responder + .pointers + .record_bytes(&target) + .await + .expect("read") + .expect("held"); + replace_records(&mut items, &target, &[held.clone(), held.clone(), held]); + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::MalformedResponse) + ); + } + + /// The pointer leaf shape is accepted only at a pointer's exact size, so it + /// cannot stand in for a chunk of another length. + #[tokio::test] + async fn a_pointer_leaf_of_any_other_length_is_refused() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let committed = responder.committed(); + let SubtreeAuditResponse::Proof { + commitment, + mut proof, + .. + } = responder.round1(nonce).await + else { + panic!("expected a proof"); + }; + let leaf = proof + .leaves + .iter_mut() + .find(|leaf| is_pointer_leaf(leaf)) + .expect("a pointer leaf"); + leaf.content_len = leaf.content_len.saturating_add(1); + + assert_eq!( + evaluate_subtree_structure( + &commitment, + &proof, + &nonce, + &committed.hash(), + &responder.peer_bytes, + ), + Err(AuditFailureReason::DigestMismatch) + ); + } +} diff --git a/src/replication/subtree.rs b/src/replication/subtree.rs index a50a628f..65a3e62d 100644 --- a/src/replication/subtree.rs +++ b/src/replication/subtree.rs @@ -499,6 +499,9 @@ pub fn build_subtree_proof( pub struct SubtreePlan { /// The selected leaves' keys, in ascending leaf-index order. pub leaf_keys: Vec, + /// For each of `leaf_keys`, whether it is committed as a pointer + /// (ADR-0016) rather than a chunk. + pub leaf_is_pointer: Vec, /// One sibling cut-hash per level on the path to the subtree root, /// root-first. pub sibling_cut_hashes: Vec<[u8; 32]>, @@ -528,11 +531,13 @@ pub fn subtree_plan( } let mut leaf_keys = Vec::with_capacity(path.real_leaf_count() as usize); + let mut leaf_is_pointer = Vec::with_capacity(path.real_leaf_count() as usize); for idx in path.leaf_start..path.leaf_end { let key = tree .key_at(idx as usize) .ok_or(BuildProofError::MissingKey { leaf_index: idx })?; leaf_keys.push(key); + leaf_is_pointer.push(tree.is_pointer_leaf(idx as usize)); } // Sibling cut-hashes, root-first. The fixed-depth slot selection no longer @@ -557,6 +562,7 @@ pub fn subtree_plan( Ok(SubtreePlan { leaf_keys, + leaf_is_pointer, sibling_cut_hashes, }) } diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index 66fe3ece..fe297705 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1023,6 +1023,39 @@ impl ChunkStore { self.files.check_capacity_for(bytes) } + /// Charge `bytes` against the disk before writing them. + /// + /// For the pointer store, which keeps its own files on this disk. Checking capacity + /// and then writing is a race the file store names explicitly: concurrent writers all + /// pass one cached measurement and cross the reserve together. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the disk cannot take `bytes` more. + pub(crate) fn reserve(&self, bytes: u64) -> Result { + self.files.reserve_bytes(bytes) + } + + /// Credit `bytes` deleted from the pointer store back to the disk budget. + /// + /// The counterpart of [`Self::reserve`]: a record the pointer store removes + /// gives its charge back, as a deleted chunk does. + pub(crate) fn release(&self, bytes: u64) { + self.files.release_bytes(bytes); + } + + /// `(written_since, in_flight)` from the capacity guard. Tests only. + #[cfg(test)] + pub(crate) fn capacity_counters(&self) -> (u64, u64) { + self.files.capacity_counters() + } + + /// What a payload of `bytes` costs the disk. Tests only. + #[cfg(test)] + pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { + crate::storage::FileStore::capacity_charge_for(bytes) + } + /// Wait until every blocking task in either backing has finished. pub async fn wait_idle(&self) { self.files.wait_idle().await; diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs index 8a6140c8..3172bdda 100644 --- a/src/storage/file_store.rs +++ b/src/storage/file_store.rs @@ -347,6 +347,14 @@ impl CapacityGuard { }) } + /// The two accounting counters, for tests that need to prove a charge was + /// settled one way rather than the other. + #[cfg(test)] + fn counters(&self) -> (u64, u64) { + let snapshot = self.snapshot.lock(); + (snapshot.written_since, snapshot.in_flight) + } + /// Give back a reservation whose write did not happen. fn release(&self, needed: u64) { let mut snapshot = self.snapshot.lock(); @@ -372,7 +380,7 @@ impl CapacityGuard { /// /// Held by whatever is actually doing the write, so the charge is released even if the /// caller's future is dropped and only the blocking closure survives. -struct Reservation { +pub(crate) struct Reservation { /// The guard this was taken from. capacity: Arc, /// Payload size, before rounding. @@ -383,7 +391,7 @@ struct Reservation { impl Reservation { /// The write landed: move the charge from in-flight to written. - fn commit(mut self) { + pub(crate) fn commit(mut self) { self.capacity.commit_reservation(self.bytes); self.settled = true; } @@ -1505,6 +1513,12 @@ impl FileStore { } } + /// Credit `len` bytes that were deleted outside this store back to the + /// cached measurement. The pointer store keeps its own files on this disk. + pub(crate) fn release_bytes(&self, len: u64) { + self.capacity.record_removed(len); + } + /// Reject work early when the disk cannot take `bytes` more. /// /// # Errors @@ -1514,6 +1528,32 @@ impl FileStore { self.capacity.check(bytes) } + /// Charge `bytes` against the disk before writing them, releasing the charge if the + /// write does not happen. + /// + /// For stores that share this disk but keep their own files — the pointer store does. + /// They must not check and then write: that is the race [`CapacityGuard::reserve`] + /// exists to close. + pub(crate) fn reserve_bytes(&self, bytes: u64) -> Result { + self.capacity.reserve(bytes) + } + + /// `(written_since, in_flight)` from the capacity guard. Tests only: it is + /// how a released charge is told apart from a stranded one. + #[cfg(test)] + pub(crate) fn capacity_counters(&self) -> (u64, u64) { + self.capacity.counters() + } + + /// What a payload of `bytes` actually costs the disk, rounded to the + /// allocation unit. Tests only, so they can assert exact counter deltas + /// rather than "it went up" — which a charge of one allocation unit would + /// satisfy for several payloads at once. + #[cfg(test)] + pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { + CapacityGuard::charge(bytes) + } + /// Force the next capacity question to re-measure the filesystem. /// /// Called after the legacy environment is removed, because that is a step change in @@ -2449,7 +2489,7 @@ fn is_windows_sharing_violation(e: &std::io::Error) -> bool { /// file for a few milliseconds after it is created, and `MoveFileEx` fails outright /// rather than queueing. Retrying a bounded number of times turns that from a failed /// write into a short pause. Every other error returns immediately. -fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { +pub(crate) fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { let mut last = match std::fs::rename(temp_path, final_path) { Ok(()) => return Ok(()), Err(e) => e, diff --git a/src/storage/handler.rs b/src/storage/handler.rs index b3c02541..840a5ef9 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -27,24 +27,25 @@ //! └─────────────────────────────────────────────────────────┘ //! ``` -#[cfg(test)] use crate::ant_protocol::DATA_TYPE_CHUNK; use crate::ant_protocol::{ settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, - ProtocolError, SettlementCompatibility, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, + ProtocolError, SettlementCompatibility, XorName, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, MAX_CHUNK_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, }; use crate::client::compute_address; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; use crate::payment::{PaymentVerifier, QuoteGenerator, VerificationContext}; +use crate::pointer::PointerService; use crate::replication::admission; use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; use crate::storage::traffic::{self, ChunkRequestKind, ChunkResponseKey}; use crate::storage::ChunkStore; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; @@ -62,7 +63,7 @@ use tokio::sync::mpsc; /// onto further peers (ADR-0002) is still accepted here, while a genuinely far /// node — which could only mis-attribute fresh-replication failures — is /// turned away. -const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; +pub const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; fn duration_ms(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) @@ -215,6 +216,46 @@ impl Drop for GetRequestTelemetry { } } +/// One latency event per pointer PUT, on the target the chunk `put_rpc` uses, +/// so the same store-latency dashboards cover both kinds. +fn log_pointer_put_rpc(elapsed: Duration, record_size: usize, response: &PointerPutResponse) { + let duration_ms = duration_ms(elapsed); + let (outcome, address): (&'static str, Option<&XorName>) = match response { + PointerPutResponse::Success { address, .. } => ("success", Some(address)), + PointerPutResponse::Unchanged { address, .. } => ("unchanged", Some(address)), + PointerPutResponse::Stale { address, .. } => ("stale", Some(address)), + PointerPutResponse::PaymentRequired { .. } => ("payment_required", None), + PointerPutResponse::Error(_) => ("error", None), + }; + let addr = address.map(hex::encode).unwrap_or_default(); + info!( + target: "ant_node::storage::rpc_latency", + duration_ms, + record_size, + outcome, + addr = %addr, + "pointer_put_rpc" + ); +} + +/// One latency event per pointer GET, beside the chunk `get_rpc`. +fn log_pointer_get_rpc(elapsed: Duration, address: &XorName, response: &PointerGetResponse) { + let duration_ms = duration_ms(elapsed); + let outcome: &'static str = match response { + PointerGetResponse::Success { .. } => "success", + PointerGetResponse::NotFound { .. } => "not_found", + PointerGetResponse::Error(_) => "error", + }; + let addr = hex::encode(address); + info!( + target: "ant_node::storage::rpc_latency", + duration_ms, + outcome, + addr = %addr, + "pointer_get_rpc" + ); +} + /// How many unversioned quote requests to receive between adoption log lines. /// /// One line per request would drown the log at production quote rates, and one @@ -302,6 +343,12 @@ pub struct AntProtocol { /// `attach_p2p_node`. Drives the self-closeness gate on client PUTs; /// `None` in unit tests that never attach a node. p2p_node: RwLock>>, + /// Serves pointer requests, when the node has a pointer store. + /// + /// `None` on a node built without one, so a peer that sends a pointer + /// message to a node that does not keep pointers gets a clean refusal + /// rather than a silent drop. + pointers: Option, } impl AntProtocol { @@ -339,9 +386,26 @@ impl AntProtocol { quote_generator, fresh_write_tx: None, p2p_node: RwLock::new(None), + pointers: None, } } + /// Serve pointer requests from `pointers`. + /// + /// Opt-in rather than built in: a node with no pointer store refuses + /// pointer messages cleanly instead of pretending to hold them. + #[must_use] + pub fn with_pointer_service(mut self, pointers: PointerService) -> Self { + self.pointers = Some(pointers); + self + } + + /// The pointer service, if this node serves pointers. + #[must_use] + pub const fn pointer_service(&self) -> Option<&PointerService> { + self.pointers.as_ref() + } + /// Attach the node's P2P handle for payment live-DHT checks. /// /// Wires the handle into the payment verifier so payment-proof closeness @@ -349,6 +413,11 @@ impl AntProtocol { /// replaces the verifier handle. pub fn attach_p2p_node(&self, node: Arc) { *self.p2p_node.write() = Some(Arc::clone(&node)); + if let Some(pointers) = &self.pointers { + // Pointers take the same self-closeness gate as chunks, judged at + // the pointer address because that is what the network routes on. + pointers.attach_p2p_node(Arc::clone(&node)); + } self.payment_verifier.attach_p2p_node(node); debug!("AntProtocol: P2PNode attached for payment live-DHT checks and self-closeness gate"); } @@ -570,6 +639,30 @@ impl AntProtocol { ), ChunkResponseKey::MerkleQuoteV2, ), + ChunkMessageBody::PointerPutRequest(req) => { + let started = Instant::now(); + let record_size = req.record.len(); + let response = match &self.pointers { + Some(service) => service.handle_put(req).await, + None => PointerPutResponse::Error(ProtocolError::StorageFailed( + "this node does not store pointers".to_string(), + )), + }; + log_pointer_put_rpc(started.elapsed(), record_size, &response); + let key = ChunkResponseKey::of_pointer_put(&response); + (ChunkMessageBody::PointerPutResponse(response), key) + } + ChunkMessageBody::PointerGetRequest(req) => { + let started = Instant::now(); + let address = req.address; + let response = match &self.pointers { + Some(service) => service.handle_get(req).await, + None => PointerGetResponse::NotFound { address }, + }; + log_pointer_get_rpc(started.elapsed(), &address, &response); + let key = ChunkResponseKey::of_pointer_get(&response); + (ChunkMessageBody::PointerGetResponse(response), key) + } // Anything else — response messages are handled by client // subscribers (e.g. send_and_await_chunk_response), not by the // protocol handler. Returning None prevents the caller from @@ -650,6 +743,22 @@ impl AntProtocol { }); } + // 2b. Refuse a chunk whose address a pointer already occupies. + // + // The mirror of the check the pointer path makes. Both kinds draw + // addresses from the same 32-byte range, and a collision — however + // infeasible — must not be resolved by whichever kind arrived second, + // because that silently destroys the other's data. + if let Some(pointers) = &self.pointers { + if pointers.store().contains(&address) { + warn!("Refusing chunk {addr_hex}: a pointer already occupies that address"); + return ChunkPutResponse::Error(ProtocolError::StorageFailed(format!( + "address {addr_hex} is already occupied by a pointer; refusing to \ + store a chunk over it" + ))); + } + } + // 3. Check if already exists (idempotent success) // // Verified against the offered bytes, not answered from the name. A name can @@ -852,17 +961,24 @@ impl AntProtocol { // Check if the chunk is already stored so we can tell the client // to skip payment (already_stored = true). + // + // Only chunks: this reads the chunk store, so asking it about any other + // kind answers a question about the wrong address space. A pointer + // quote names a state, not content, and every pointer state — creation + // or update — is paid for, so it is never already stored. + // // The match intentionally logs the error when the `logging` feature is // active. Clippy suggests `unwrap_or_default()` when logging is compiled // out, but keeping the explicit match preserves the diagnostic intent. #[allow(clippy::manual_unwrap_or_default)] - let already_stored = match self.storage.exists(&request.address) { - Ok(exists) => exists, - Err(e) => { - warn!("Storage check failed for {addr_hex}: {e}"); - false // Assume not stored on error — generate a normal quote. - } - }; + let already_stored = request.data_type == DATA_TYPE_CHUNK + && match self.storage.exists(&request.address) { + Ok(exists) => exists, + Err(e) => { + warn!("Storage check failed for {addr_hex}: {e}"); + false // Assume not stored on error — generate a normal quote. + } + }; if already_stored { debug!("Chunk {addr_hex} already stored — returning quote with already_stored=true"); @@ -1090,6 +1206,7 @@ mod tests { use super::*; use crate::payment::metrics::QuotingMetricsTracker; use crate::payment::{EvmVerifierConfig, PaymentVerifierConfig}; + use crate::pointer::{DATA_TYPE_POINTER, POINTER_WIRE_LEN}; use crate::storage::ChunkStoreConfig; use evmlib::RewardsAddress; use saorsa_core::identity::NodeIdentity; @@ -1945,6 +2062,56 @@ mod tests { ); } + #[tokio::test] + async fn a_stored_chunk_does_not_suppress_a_pointer_quote() { + // A pointer quote names a state, not content. If a chunk sitting at + // that address could set `already_stored`, a majority of nodes would + // tell the client to skip payment and the write would then be refused + // as unpaid. Only chunks may answer from the chunk store. + let (protocol, _temp) = create_test_protocol().await; + + let content = b"a chunk that shares a pointer state address"; + let address = ChunkStore::compute_address(content); + protocol.payment_verifier().cache_insert(address); + let put_msg = ChunkMessage { + request_id: 320, + body: ChunkMessageBody::PutRequest(ChunkPutRequest::new( + address, + Bytes::copy_from_slice(content), + )), + }; + let put_bytes = put_msg.encode().expect("encode put"); + let _ = protocol + .try_handle_request(&put_bytes) + .await + .expect("handle put"); + + let quote_msg = ChunkMessage { + request_id: 321, + body: ChunkMessageBody::QuoteRequest(ChunkQuoteRequest { + address, + data_size: POINTER_WIRE_LEN as u64, + data_type: DATA_TYPE_POINTER, + }), + }; + let quote_bytes = quote_msg.encode().expect("encode quote"); + let response_bytes = protocol + .try_handle_request("e_bytes) + .await + .expect("handle quote") + .expect("expected response"); + + match ChunkMessage::decode(&response_bytes).expect("decode").body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { + already_stored, .. + }) => assert!( + !already_stored, + "a chunk must not answer for a pointer state" + ), + other => panic!("expected a quote, got: {other:?}"), + } + } + #[tokio::test] async fn test_quote_already_stored_flag() { let (protocol, _temp) = create_test_protocol().await; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index b60d71c7..fc493393 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -57,9 +57,11 @@ pub(crate) mod traffic; pub use crate::ant_protocol::XorName; pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; +pub(crate) use file_store::{rename_with_retry, Reservation}; pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; +pub use handler::SELF_CLOSENESS_GATE_WIDTH; pub(crate) use lmdb::CapacityVerdict; pub use lmdb::{LmdbStorage, LmdbStorageConfig}; pub use migration::{MigrationConfig, MigrationPhase, MigrationState}; diff --git a/src/storage/traffic.rs b/src/storage/traffic.rs index dc316f4e..35688e8f 100644 --- a/src/storage/traffic.rs +++ b/src/storage/traffic.rs @@ -13,6 +13,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use crate::ant_protocol::{ChunkGetResponse, ChunkMessageBody, ChunkPutResponse}; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; /// Kind of an inbound chunk message, for the rx table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -23,6 +24,8 @@ pub enum ChunkRequestKind { MerkleQuote, QuoteV2, MerkleQuoteV2, + PointerGet, + PointerPut, /// A non-request variant (responses meant for client subscribers) or an /// unknown future variant. Other, @@ -31,7 +34,7 @@ pub enum ChunkRequestKind { } impl ChunkRequestKind { - const N: usize = 8; + const N: usize = 10; const fn index(self) -> usize { match self { @@ -41,8 +44,10 @@ impl ChunkRequestKind { Self::MerkleQuote => 3, Self::QuoteV2 => 4, Self::MerkleQuoteV2 => 5, - Self::Other => 6, - Self::DecodeError => 7, + Self::PointerGet => 6, + Self::PointerPut => 7, + Self::Other => 8, + Self::DecodeError => 9, } } } @@ -61,13 +66,21 @@ pub enum ChunkResponseKey { MerkleQuote, QuoteV2, MerkleQuoteV2, + PointerGetSuccess, + PointerGetNotFound, + PointerGetError, + PointerPutSuccess, + PointerPutUnchanged, + PointerPutStale, + PointerPutPaymentRequired, + PointerPutError, /// A response variant this table does not itemise (e.g. an `Other` /// outcome on a `#[non_exhaustive]` enum). Other, } impl ChunkResponseKey { - const N: usize = 12; + const N: usize = 20; const fn index(self) -> usize { match self { @@ -82,7 +95,15 @@ impl ChunkResponseKey { Self::MerkleQuote => 8, Self::QuoteV2 => 9, Self::MerkleQuoteV2 => 10, - Self::Other => 11, + Self::PointerGetSuccess => 11, + Self::PointerGetNotFound => 12, + Self::PointerGetError => 13, + Self::PointerPutSuccess => 14, + Self::PointerPutUnchanged => 15, + Self::PointerPutStale => 16, + Self::PointerPutPaymentRequired => 17, + Self::PointerPutError => 18, + Self::Other => 19, } } } @@ -97,6 +118,8 @@ impl ChunkRequestKind { ChunkMessageBody::MerkleCandidateQuoteRequest(_) => Self::MerkleQuote, ChunkMessageBody::QuoteRequestV2(_) => Self::QuoteV2, ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) => Self::MerkleQuoteV2, + ChunkMessageBody::PointerGetRequest(_) => Self::PointerGet, + ChunkMessageBody::PointerPutRequest(_) => Self::PointerPut, _ => Self::Other, } } @@ -113,6 +136,34 @@ impl ChunkResponseKey { } } + /// Classify a pointer GET response by outcome. + /// + /// Exhaustive, unlike the chunk classifiers: the pointer response enums are + /// not `#[non_exhaustive]`, so a new variant is a compile error here rather + /// than silently becoming `Other`. + pub fn of_pointer_get(response: &PointerGetResponse) -> Self { + match response { + PointerGetResponse::Success { .. } => Self::PointerGetSuccess, + PointerGetResponse::NotFound { .. } => Self::PointerGetNotFound, + PointerGetResponse::Error(_) => Self::PointerGetError, + } + } + + /// Classify a pointer PUT response by outcome. + /// + /// `Unchanged` and `Stale` are itemised separately from `Success` because + /// they are what a re-submission and a lost race look like, and telling + /// those apart is the point of counting paid writes at all. + pub fn of_pointer_put(response: &PointerPutResponse) -> Self { + match response { + PointerPutResponse::Success { .. } => Self::PointerPutSuccess, + PointerPutResponse::Unchanged { .. } => Self::PointerPutUnchanged, + PointerPutResponse::Stale { .. } => Self::PointerPutStale, + PointerPutResponse::PaymentRequired { .. } => Self::PointerPutPaymentRequired, + PointerPutResponse::Error(_) => Self::PointerPutError, + } + } + /// Classify a PUT response by outcome. pub fn of_put(response: &ChunkPutResponse) -> Self { match response { @@ -160,10 +211,11 @@ pub fn record_send_failed(bytes: usize) { /// Emit the cumulative chunk-RPC traffic as INFO summary lines, target /// `ant_node::storage::traffic`. /// -/// Flat snake-case keys like the replication summary. Two lines sharing the +/// Flat snake-case keys like the replication summary. Three lines sharing the /// same target and message, distinguished by `group`: rx by request kind -/// (`group = 1`) and tx by kind × outcome (`group = 2`), keeping each under -/// `tracing`'s 32-field cap. +/// (`group = 1`), chunk and quote tx by kind × outcome (`group = 2`), and +/// pointer tx by kind × outcome (`group = 3`). Pointers take a line of their +/// own because `group = 2` is already close to `tracing`'s 32-field cap. pub fn log_chunk_rpc_traffic_summary() { use ChunkRequestKind as Q; use ChunkResponseKey as R; @@ -183,6 +235,8 @@ pub fn log_chunk_rpc_traffic_summary() { quote_v2_rx_bytes = rb(Q::QuoteV2), quote_v2_rx_count = rc(Q::QuoteV2), merkle_quote_v2_rx_bytes = rb(Q::MerkleQuoteV2), merkle_quote_v2_rx_count = rc(Q::MerkleQuoteV2), + pointer_get_rx_bytes = rb(Q::PointerGet), pointer_get_rx_count = rc(Q::PointerGet), + pointer_put_rx_bytes = rb(Q::PointerPut), pointer_put_rx_count = rc(Q::PointerPut), other_rx_bytes = rb(Q::Other), other_rx_count = rc(Q::Other), decode_error_rx_bytes = rb(Q::DecodeError), decode_error_rx_count = rc(Q::DecodeError), "chunk rpc traffic summary (cumulative)" @@ -210,6 +264,28 @@ pub fn log_chunk_rpc_traffic_summary() { send_failed_tx_count = SEND_FAILED_COUNT.load(Ordering::Relaxed), "chunk rpc traffic summary (cumulative)" ); + + crate::logging::info!( + target: "ant_node::storage::traffic", + group = 3, + pointer_get_success_tx_bytes = tb(R::PointerGetSuccess), + pointer_get_success_tx_count = tc(R::PointerGetSuccess), + pointer_get_not_found_tx_bytes = tb(R::PointerGetNotFound), + pointer_get_not_found_tx_count = tc(R::PointerGetNotFound), + pointer_get_error_tx_bytes = tb(R::PointerGetError), + pointer_get_error_tx_count = tc(R::PointerGetError), + pointer_put_success_tx_bytes = tb(R::PointerPutSuccess), + pointer_put_success_tx_count = tc(R::PointerPutSuccess), + pointer_put_unchanged_tx_bytes = tb(R::PointerPutUnchanged), + pointer_put_unchanged_tx_count = tc(R::PointerPutUnchanged), + pointer_put_stale_tx_bytes = tb(R::PointerPutStale), + pointer_put_stale_tx_count = tc(R::PointerPutStale), + pointer_put_payment_required_tx_bytes = tb(R::PointerPutPaymentRequired), + pointer_put_payment_required_tx_count = tc(R::PointerPutPaymentRequired), + pointer_put_error_tx_bytes = tb(R::PointerPutError), + pointer_put_error_tx_count = tc(R::PointerPutError), + "chunk rpc traffic summary (cumulative)" + ); } #[cfg(test)] @@ -225,6 +301,8 @@ mod tests { ChunkRequestKind::MerkleQuote, ChunkRequestKind::QuoteV2, ChunkRequestKind::MerkleQuoteV2, + ChunkRequestKind::PointerGet, + ChunkRequestKind::PointerPut, ChunkRequestKind::Other, ChunkRequestKind::DecodeError, ]; @@ -245,6 +323,14 @@ mod tests { ChunkResponseKey::MerkleQuote, ChunkResponseKey::QuoteV2, ChunkResponseKey::MerkleQuoteV2, + ChunkResponseKey::PointerGetSuccess, + ChunkResponseKey::PointerGetNotFound, + ChunkResponseKey::PointerGetError, + ChunkResponseKey::PointerPutSuccess, + ChunkResponseKey::PointerPutUnchanged, + ChunkResponseKey::PointerPutStale, + ChunkResponseKey::PointerPutPaymentRequired, + ChunkResponseKey::PointerPutError, ChunkResponseKey::Other, ]; let mut seen = std::collections::HashSet::new(); diff --git a/src/web_rtc.rs b/src/web_rtc.rs index 8c421ca3..6bb3e8e0 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -58,6 +58,11 @@ const MAX_FIND_NODE_RESULTS: usize = 20; // target; clients still obey their global memory/CPU admission budgets. const MAX_CHANNEL_REQUESTS: usize = 4; const RPC_MULTIPLEX_CAPABILITY: &str = "rpc-multiplex-4"; + +/// Advertised when `chunk_protocol` admits pointer reads and paid pointer +/// writes (ADR-0016), so a browser client can tell a node that would refuse +/// them from one that failed. +const POINTER_PROTOCOL_CAPABILITY: &str = "pointer_protocol"; // Browser dials use a 10-second channel-open timeout. Give successful clients // modest server-side headroom while bounding associations that never open one. const FIRST_DATA_CHANNEL_TIMEOUT: Duration = Duration::from_secs(15); @@ -1732,7 +1737,14 @@ async fn process_request( // PUT acknowledgements and signed quotes contain no chunk payload. 64 KiB // covers the ML-DSA public key/signature, quote and signed commitment, including // worst-case MessagePack integer encoding. GET alone needs a full wire buffer. +// +// A pointer read returns one record of `POINTER_WIRE_LEN` bytes, so it is small +// too and needs no bulk slot (see `request_is_get`). const SMALL_BINARY_RESPONSE_BYTES: usize = 64 * 1024; +const _: () = assert!( + ant_protocol::pointer::POINTER_WIRE_LEN < SMALL_BINARY_RESPONSE_BYTES / 2, + "a pointer read must fit the small response bound with room for its envelope" +); fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { match body { @@ -1741,7 +1753,9 @@ fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { | ChunkMessageBody::QuoteRequest(_) | ChunkMessageBody::QuoteRequestV2(_) | ChunkMessageBody::MerkleCandidateQuoteRequest(_) - | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), + | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) + | ChunkMessageBody::PointerGetRequest(_) + | ChunkMessageBody::PointerPutRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), _ => Err("unsupported chunk protocol request".to_string()), } } @@ -1776,20 +1790,35 @@ fn hello_response(request_id: u64, state: &ServerState) -> Response { max_chunk_size: MAX_CHUNK_SIZE, endpoint, payment: state.payment.clone(), - capabilities: vec![ - "chunk_protocol".into(), - RPC_MULTIPLEX_CAPABILITY.into(), - "find_node".into(), - ant_protocol::transport::ADDRESS_V2_CAPABILITY.into(), - "get_chunk".into(), - "quote_chunk".into(), - "put_chunk".into(), - ], + capabilities: hello_capabilities(serves_pointers(state.ant_protocol.as_deref())), }, 0, ) } +/// Whether requests reach a pointer store. A node without one refuses pointer +/// writes and has none to read, so it must not invite them: a browser could +/// otherwise pay a quote before learning the write goes nowhere. +fn serves_pointers(protocol: Option<&AntProtocol>) -> bool { + protocol.is_some_and(|protocol| protocol.pointer_service().is_some()) +} + +fn hello_capabilities(pointers: bool) -> Vec { + let mut capabilities: Vec = vec![ + "chunk_protocol".into(), + RPC_MULTIPLEX_CAPABILITY.into(), + "find_node".into(), + ant_protocol::transport::ADDRESS_V2_CAPABILITY.into(), + "get_chunk".into(), + "quote_chunk".into(), + "put_chunk".into(), + ]; + if pointers { + capabilities.push(POINTER_PROTOCOL_CAPABILITY.into()); + } + capabilities +} + async fn process_find_node( request_id: u64, target: String, @@ -2257,6 +2286,10 @@ struct ServerState { )] mod tests { use super::*; + use ant_protocol::chunk::{ + PointerGetRequest, PointerGetResponse, PointerPutRequest, PointerPutResponse, + }; + use ant_protocol::pointer::POINTER_WIRE_LEN; use std::net::Ipv4Addr; #[test] @@ -2745,6 +2778,57 @@ mod tests { } } + /// Pointers are advertised only by a node that serves them. + #[test] + fn only_a_node_that_serves_pointers_advertises_them() { + let advertises = |caps: &[String]| caps.iter().any(|c| c == POINTER_PROTOCOL_CAPABILITY); + assert!(advertises(&hello_capabilities(true))); + assert!(!advertises(&hello_capabilities(false))); + assert!(!serves_pointers(None), "no storage, no pointers"); + } + + /// Pointer reads and paid pointer writes are admitted (ADR-0016), and a + /// full record fits the small bound they are given. + #[test] + fn pointer_requests_are_admitted_and_a_full_record_fits() { + let get = ChunkMessageBody::PointerGetRequest(PointerGetRequest::new([0xff; 32])); + let put = ChunkMessageBody::PointerPutRequest(PointerPutRequest::with_payment( + vec![0xff; POINTER_WIRE_LEN].into(), + vec![0xff; 16 * 1024], + )); + assert_eq!( + binary_response_limit(&get).expect("pointer GET admitted"), + SMALL_BINARY_RESPONSE_BYTES + ); + assert_eq!( + binary_response_limit(&put).expect("pointer PUT admitted"), + SMALL_BINARY_RESPONSE_BYTES + ); + + let record = ChunkMessage { + request_id: u64::MAX, + body: ChunkMessageBody::PointerGetResponse(PointerGetResponse::Success { + record: vec![0xff; POINTER_WIRE_LEN].into(), + }), + }; + let bytes = encode_binary_response(&record, SMALL_BINARY_RESPONSE_BYTES) + .expect("a full record fits"); + assert_eq!(bytes, record.encode().expect("shared wire encoding")); + + let stored = ChunkMessage { + request_id: u64::MAX, + body: ChunkMessageBody::PointerPutResponse(PointerPutResponse::Success { + address: [0xff; 32], + state_id: [0xff; 32], + }), + }; + assert!(encode_binary_response(&stored, SMALL_BINARY_RESPONSE_BYTES).is_ok()); + + // Replies are never admitted as requests. + assert!(binary_response_limit(&record.body).is_err()); + assert!(binary_response_limit(&stored.body).is_err()); + } + #[test] fn maximum_binary_get_fits_and_oversized_responses_are_rejected() { let request = ChunkMessageBody::GetRequest(ant_protocol::ChunkGetRequest::new([0xff; 32])); diff --git a/src/web_rtc/errors.rs b/src/web_rtc/errors.rs index a24d20d3..37227f00 100644 --- a/src/web_rtc/errors.rs +++ b/src/web_rtc/errors.rs @@ -5,6 +5,7 @@ use super::ServerResult; use crate::logging::warn; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use ant_protocol::{ ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutResponse, ChunkQuoteResponse, MerkleCandidateQuoteResponse, ProtocolError, @@ -45,10 +46,13 @@ pub(super) fn sanitize_response(message: &mut ChunkMessage) -> ServerResult<()> | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(error)) | ChunkMessageBody::MerkleCandidateQuoteResponse(MerkleCandidateQuoteResponse::Error( error, - )) => { + )) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(error)) + | ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(error)) => { sanitize_protocol_error(error); } - ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { message }) => { + ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { message }) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { message }) => { warn!(detail = %message, "Browser payment verification failed"); *message = "valid payment is required".to_string(); } @@ -61,7 +65,17 @@ pub(super) fn sanitize_response(message: &mut ChunkMessage) -> ServerResult<()> | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { .. }) | ChunkMessageBody::MerkleCandidateQuoteResponse(MerkleCandidateQuoteResponse::Success { .. - }) => {} + }) + // A pointer outcome names only the address and a state identifier, + // and a record is owner-signed public data. + | ChunkMessageBody::PointerPutResponse( + PointerPutResponse::Success { .. } + | PointerPutResponse::Unchanged { .. } + | PointerPutResponse::Stale { .. }, + ) + | ChunkMessageBody::PointerGetResponse( + PointerGetResponse::Success { .. } | PointerGetResponse::NotFound { .. }, + ) => {} // New response variants must explicitly opt into the browser boundary. _ => { return Err(public_error( @@ -121,6 +135,8 @@ mod tests { ChunkMessageBody::MerkleCandidateQuoteResponse( MerkleCandidateQuoteResponse::Error(error.clone()), ), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(error.clone())), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(error.clone())), ]; for body in responses { let original = ChunkMessage { @@ -139,7 +155,10 @@ mod tests { | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(sanitized)) | ChunkMessageBody::MerkleCandidateQuoteResponse( MerkleCandidateQuoteResponse::Error(sanitized), - )) = &decoded.body + ) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(sanitized)) + | ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(sanitized))) = + &decoded.body else { unreachable!(); }; @@ -166,6 +185,16 @@ mod tests { assert!( matches!(decoded.body, ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { ref message }) if message == "valid payment is required") ); + let pointer = ChunkMessage { + request_id: 7, + body: ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { + message: PRIVATE_DETAIL.into(), + }), + }; + let decoded = decode_response(pointer.encode().expect("encode").into()).expect("redact"); + assert!( + matches!(decoded.body, ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { ref message }) if message == "valid payment is required") + ); for code in [ "storage_error", "quote_failed", @@ -186,6 +215,22 @@ mod tests { address: [1; 32], content: vec![2; 128], }), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::Success { + record: vec![3; 64].into(), + }), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::NotFound { address: [4; 32] }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Success { + address: [5; 32], + state_id: [6; 32], + }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Unchanged { + address: [5; 32], + state_id: [6; 32], + }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Stale { + address: [5; 32], + state_id: [7; 32], + }), ChunkMessageBody::PutResponse(ChunkPutResponse::Error( ProtocolError::AddressMismatch { expected: [1; 32], diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index 39db7c5f..eaa560f9 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -4,13 +4,50 @@ //! both the ant node network and optional Anvil EVM testnet. use super::anvil::TestAnvil; -use super::testnet::{TestNetwork, TestNetworkConfig, TestNode}; +use super::testnet::{TestNetwork, TestNetworkConfig, TestNode, TestnetError}; use ant_node::client::XorName; use evmlib::common::TxHash; use saorsa_core::P2PNode; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -use tracing::info; +use tracing::{info, warn}; + +/// How many port ranges a network is brought up on before its bind failure +/// is reported. +/// +/// A node retries its own port, which clears a socket still being released. +/// It cannot clear a port the host will not hand out at all, as happens on +/// Windows runners whose reserved port ranges move between runs, so a network +/// that cannot create a node moves to a fresh random range instead. +const NETWORK_START_ATTEMPTS: u32 = 3; + +/// Create and start a network from `config`, moving it to a fresh port range +/// if a node cannot be created. +async fn start_network(mut config: TestNetworkConfig) -> Result { + let mut attempt = 1; + loop { + let mut network = TestNetwork::new(config.clone()).await?; + match network.start().await { + Ok(()) => return Ok(network), + Err(TestnetError::Startup(reason)) + if attempt < NETWORK_START_ATTEMPTS + && reason.starts_with("Failed to create node ") => + { + let base_port = config.base_port; + warn!( + "Test network on base port {base_port} could not create a node \ + ({reason}); retrying on a fresh port range" + ); + if let Err(e) = network.shutdown().await { + warn!("Cleanup after a failed bring-up failed: {e}"); + } + config = config.with_fresh_ports(); + attempt += 1; + } + Err(e) => return Err(e.into()), + } + } +} /// Error type for test harness operations. #[derive(Debug, thiserror::Error)] @@ -192,8 +229,7 @@ impl TestHarness { pub async fn setup_with_config(config: TestNetworkConfig) -> Result { info!("Setting up test harness with {} nodes", config.node_count); - let mut network = TestNetwork::new(config).await?; - network.start().await?; + let network = start_network(config).await?; Ok(Self { network, @@ -257,8 +293,7 @@ impl TestHarness { config.node_count ); - let mut network = TestNetwork::new(config).await?; - network.start().await?; + let network = start_network(config).await?; // Warm up DHT routing tables (essential for quote collection) info!("Warming up DHT routing tables..."); diff --git a/tests/e2e/integration_tests.rs b/tests/e2e/integration_tests.rs index 8369d445..3f65ba0c 100644 --- a/tests/e2e/integration_tests.rs +++ b/tests/e2e/integration_tests.rs @@ -11,6 +11,7 @@ use super::testnet::{ use super::{NetworkState, TestHarness, TestNetwork, TestNetworkConfig}; use saorsa_core::P2PEvent; use serial_test::serial; +use std::net::UdpSocket; use std::time::Duration; /// Test that a minimal network (5 nodes) can form and stabilize. @@ -273,3 +274,24 @@ async fn test_node_to_node_messaging() { .await .expect("Failed to teardown test harness"); } + +/// A network whose first node cannot bind its port is brought up on a fresh +/// port range rather than failing the test that asked for it. +#[tokio::test] +#[serial] +async fn a_network_whose_port_is_taken_moves_to_a_fresh_range() { + let config = TestNetworkConfig::minimal(); + let taken = config.base_port; + let _holder = UdpSocket::bind(("127.0.0.1", taken)).expect("hold the first node's port"); + + let harness = TestHarness::setup_with_config(config) + .await + .expect("the network comes up on another range"); + assert_ne!( + harness.network().config().base_port, + taken, + "the network moved off the port it could not bind" + ); + + harness.teardown().await.expect("teardown"); +} diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index 74d34637..ec2a74c7 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -60,6 +60,9 @@ mod merkle_payment; #[cfg(test)] mod replication; +// Pointer replication across real nodes (ADR-0016). +mod pointer_replication; + #[cfg(test)] mod security_attacks; diff --git a/tests/e2e/pointer_replication.rs b/tests/e2e/pointer_replication.rs new file mode 100644 index 00000000..c031d719 --- /dev/null +++ b/tests/e2e/pointer_replication.rs @@ -0,0 +1,739 @@ +//! Pointer replication across real nodes (ADR-0016). +//! +//! Each test runs a live network whose nodes carry the same pointer service and +//! replication engine a production node does, over real QUIC. Payment is +//! pre-marked in each node's verifier cache — the chain is not what is under +//! test here — but every other check a node makes runs for real: signatures, +//! responsibility, the merge rule, quorum and the capability gate. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::testnet::{TestNetworkConfig, TestNode}; +use super::TestHarness; +use ant_node::ant_protocol::chunk::{ + ChunkMessage, ChunkMessageBody, PointerPutRequest, PointerPutResponse, +}; +use ant_node::pointer::PointerStore; +use ant_node::replication::audit::AuditTickResult; +use ant_node::replication::commitment::pointer_leaf_hash; +use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; +use ant_node::replication::pointer::{PointerFreshWrite, PointerReplication}; +use ant_node::ReplicationConfig; +use ant_protocol::pointer::{Pointer, PointerState, PointerTarget, PointerTargetKind}; +use bytes::Bytes; +use saorsa_core::identity::PeerId; +use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; +use serial_test::serial; +use std::sync::Arc; +use std::time::Duration; + +/// How long to wait for replication to reach a node. +const SETTLE: Duration = Duration::from_secs(30); + +/// How often to look while waiting. +const POLL: Duration = Duration::from_millis(200); + +/// A proof the receivers never parse: the state is pre-marked as paid in each +/// verifier's cache, so verification answers from the cache. +const DUMMY_PROOF: [u8; 64] = [0x01; 64]; + +fn owner() -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair().expect("keypair") +} + +fn signed(pk: &MlDsaPublicKey, sk: &MlDsaSecretKey, counter: u64, target: u8) -> Pointer { + Pointer::sign( + sk, + pk, + counter, + PointerTarget::new(PointerTargetKind::Chunk, [target; 32]), + ) + .expect("sign") +} + +fn store(node: &TestNode) -> PointerStore { + node.ant_protocol + .as_ref() + .expect("protocol") + .pointer_service() + .expect("pointer service") + .store() + .clone() +} + +fn replication(node: &TestNode) -> &Arc { + node.replication_engine + .as_ref() + .expect("engine") + .pointer_replication() + .expect("pointer replication") +} + +fn commitments(node: &TestNode) -> &ResponderCommitmentState { + node.replication_engine + .as_ref() + .expect("engine") + .commitment_state() +} + +fn peer(node: &TestNode) -> PeerId { + *node.p2p_node.as_ref().expect("p2p").peer_id() +} + +fn held(node: &TestNode, record: &Pointer) -> Option { + store(node).state(&record.address()) +} + +fn holds(node: &TestNode, record: &Pointer) -> bool { + held(node, record).is_some_and(|state| state.state_id == record.state_id()) +} + +/// Mark `record`'s state as paid on every node, as a settled payment would. +fn mark_paid(harness: &TestHarness, record: &Pointer) { + for i in 0..harness.node_count() { + if let Some(protocol) = harness.test_node(i).and_then(|n| n.ant_protocol.as_ref()) { + protocol + .payment_verifier() + .cache_insert_pointer(record.address(), record.state_id()); + } + } +} + +/// Every node's index, except `except`. +fn others(harness: &TestHarness, except: &[usize]) -> Vec { + (0..harness.node_count()) + .filter(|i| !except.contains(i)) + .collect() +} + +/// Wait until `node` holds exactly `record`'s state. +async fn wait_for(harness: &TestHarness, index: usize, record: &Pointer) -> bool { + let deadline = tokio::time::Instant::now() + SETTLE; + while tokio::time::Instant::now() < deadline { + if harness + .test_node(index) + .is_some_and(|node| holds(node, record)) + { + return true; + } + tokio::time::sleep(POLL).await; + } + false +} + +/// Send a paid pointer PUT to one node through its request handler, as a +/// client's PUT arrives, and return its answer. +async fn put(node: &TestNode, record: &Pointer) -> PointerPutResponse { + let message = ChunkMessage { + request_id: 1, + body: ChunkMessageBody::PointerPutRequest(PointerPutRequest::with_payment( + Bytes::from(record.to_bytes()), + DUMMY_PROOF.to_vec(), + )), + }; + let reply = node + .ant_protocol + .as_ref() + .expect("protocol") + .try_handle_request(&message.encode().expect("encode")) + .await + .expect("handle") + .expect("answered"); + match ChunkMessage::decode(&reply).expect("decode").body { + ChunkMessageBody::PointerPutResponse(response) => response, + other => panic!("expected a pointer PUT response, got {other:?}"), + } +} + +/// Have every node in `from` push its hints to every node in `to`, so each +/// learns the others understand pointers and hears what they hold. +async fn exchange_hints(harness: &TestHarness, from: &[usize], to: &[usize]) { + let targets: Vec = to + .iter() + .filter_map(|i| harness.test_node(*i).map(peer)) + .collect(); + for i in from { + let node = harness.test_node(*i).expect("node"); + let own = peer(node); + let peers: Vec = targets.iter().copied().filter(|p| *p != own).collect(); + replication(node).push_hints(&peers).await; + } + // One-way messages: give them a moment to land. + tokio::time::sleep(Duration::from_millis(500)).await; +} + +/// A client's paid PUT reaches one node, and replication carries it to every +/// other member of the close group: the node that accepted it offers it on, +/// with the proof, and each receiver verifies and stores it. +#[tokio::test] +#[serial] +async fn a_paid_put_to_one_node_reaches_its_whole_close_group() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 0, 1); + mark_paid(&harness, &record); + + let entry = 3; + match put(harness.test_node(entry).expect("node"), &record).await { + PointerPutResponse::Success { state_id, .. } => assert_eq!(state_id, record.state_id()), + other => panic!("the entry node refused the PUT: {other:?}"), + } + + // In a five-node network every node is in every close group. + for i in others(&harness, &[entry]) { + assert!( + wait_for(&harness, i, &record).await, + "node {i} never received the pointer" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// An update written to one node replaces the old state on every node. +#[tokio::test] +#[serial] +async fn an_update_replicates_and_replaces_the_old_state_everywhere() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let created = signed(&pk, &sk, 0, 1); + let updated = signed(&pk, &sk, 4, 2); + mark_paid(&harness, &created); + mark_paid(&harness, &updated); + + put(harness.test_node(2).expect("node"), &created).await; + for i in 0..harness.node_count() { + assert!( + wait_for(&harness, i, &created).await, + "node {i} lacks the create" + ); + } + put(harness.test_node(4).expect("node"), &updated).await; + for i in 0..harness.node_count() { + assert!( + wait_for(&harness, i, &updated).await, + "node {i} still holds the old state" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// A node that missed an update is brought level by neighbour sync: the +/// holders hint the newer state, it asks the group which state each holds, +/// adopts the one a quorum hold, and fetches it from one of them. +#[tokio::test] +#[serial] +async fn a_node_that_missed_an_update_is_repaired_by_neighbour_sync() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let created = signed(&pk, &sk, 0, 1); + let missed = signed(&pk, &sk, 3, 2); + let lagging = 4; + let current = others(&harness, &[lagging]); + + // Everyone holds the create; everyone but the laggard holds the update, + // written straight into their stores so nothing replicates it. + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&created.to_bytes()) + .await + .expect("put"); + } + for i in ¤t { + store(harness.test_node(*i).expect("node")) + .put_bytes(&missed.to_bytes()) + .await + .expect("put"); + } + assert!(holds(harness.test_node(lagging).expect("node"), &created)); + + // Nothing but hints reaches the laggard: no fresh write carries the + // update, so holding it afterwards means it was repaired. The engine's + // own verification loop runs too; driving a round here only saves waiting. + exchange_hints(&harness, ¤t, &[lagging]).await; + replication(harness.test_node(lagging).expect("node")) + .verify_due() + .await; + + assert!( + wait_for(&harness, lagging, &missed).await, + "the lagging node never caught up" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node that joins after a pointer was written obtains it, with nothing but +/// the engine's own loops: its bootstrap sync reaches its neighbours, they +/// push their hints back, and it verifies and fetches. +#[tokio::test] +#[serial] +async fn a_node_that_joins_later_obtains_the_existing_pointers() { + let mut harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 2, 1); + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + + let joined = harness.add_node().await.expect("add node"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + let mut obtained = false; + while tokio::time::Instant::now() < deadline { + if harness + .test_node(joined) + .is_some_and(|node| holds(node, &record)) + { + obtained = true; + break; + } + // Stand in for the periodic timer, which runs every ten minutes. + for i in 0..harness.node_count() { + if let Some(engine) = harness + .test_node(i) + .and_then(|n| n.replication_engine.as_ref()) + { + engine.trigger_neighbor_sync(); + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + assert!(obtained, "the joining node never obtained the pointer"); + + harness.teardown().await.expect("teardown"); +} + +/// A newer state that only one node holds is not adopted by the rest, however +/// well signed: one peer's word is not the network's. It is exactly what an +/// owner who got one node to store an unpaid state would try. +#[tokio::test] +#[serial] +async fn a_state_only_one_node_holds_is_not_adopted() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let agreed = signed(&pk, &sk, 1, 1); + let lone = signed(&pk, &sk, 9, 9); + let lone_holder = 0; + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&agreed.to_bytes()) + .await + .expect("put"); + } + store(harness.test_node(lone_holder).expect("node")) + .put_bytes(&lone.to_bytes()) + .await + .expect("put"); + + let rest = others(&harness, &[lone_holder]); + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &everyone).await; + for i in &rest { + replication(harness.test_node(*i).expect("node")) + .verify_due() + .await; + } + tokio::time::sleep(Duration::from_secs(2)).await; + + let lone_peer = peer(harness.test_node(lone_holder).expect("node")); + for i in &rest { + let node = harness.test_node(*i).expect("node"); + // The lone holder's hints did arrive, so what follows is the quorum + // refusing, not a hint that never came. + assert!( + replication(node).is_capable(&lone_peer), + "node {i} never heard from the lone holder" + ); + assert!( + holds(node, &agreed), + "node {i} adopted a state only one node holds" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// A fresh offer whose state was never paid for is refused by every receiver. +#[tokio::test] +#[serial] +async fn a_fresh_offer_that_was_not_paid_for_is_refused() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let unpaid = signed(&pk, &sk, 0, 1); + let source = 3; + let node = harness.test_node(source).expect("node"); + store(node) + .put_bytes(&unpaid.to_bytes()) + .await + .expect("put"); + replication(node) + .replicate_fresh(PointerFreshWrite { + record: unpaid.to_bytes(), + payment_proof: DUMMY_PROOF.to_vec(), + }) + .await; + tokio::time::sleep(Duration::from_secs(5)).await; + + for i in others(&harness, &[source]) { + assert!( + held(harness.test_node(i).expect("node"), &unpaid).is_none(), + "node {i} stored an unpaid pointer" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// Some minutes after offering a fresh state, the offering node asks each +/// close-group member for it. A member that cannot produce it is penalised, +/// and one that can is not. +#[tokio::test] +#[serial] +async fn the_possession_check_penalises_only_a_member_that_dropped_the_record() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 0, 1); + let checker = 3; + let dropper = 1; + let keeper = 2; + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + store(harness.test_node(dropper).expect("node")) + .delete(&record.address()) + .await + .expect("delete"); + + // The checker only asks peers that understand pointers. + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &[checker]).await; + + let checker_node = harness.test_node(checker).expect("node"); + let checker_p2p = checker_node.p2p_node.as_ref().expect("p2p"); + let dropper_peer = peer(harness.test_node(dropper).expect("node")); + let keeper_peer = peer(harness.test_node(keeper).expect("node")); + let dropper_before = checker_p2p.peer_trust(&dropper_peer); + let keeper_before = checker_p2p.peer_trust(&keeper_peer); + + replication(checker_node) + .check_possession(record.state(), &[dropper_peer, keeper_peer]) + .await; + + assert!( + checker_p2p.peer_trust(&dropper_peer) < dropper_before, + "the member that dropped the record was not penalised" + ); + assert!( + checker_p2p.peer_trust(&keeper_peer) >= keeper_before, + "the member that holds the record was penalised" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A network whose close group is two nodes, so that in five nodes some node +/// is always outside the retention width (two plus the margin of two) for any +/// address, with no pruning hysteresis. +fn prune_network(paid_width: usize) -> TestNetworkConfig { + TestNetworkConfig { + replication_config: Some(ReplicationConfig { + close_group_size: 2, + // The quorum may not exceed the group. + quorum_threshold: 2, + paid_list_close_group_size: paid_width, + prune_hysteresis_duration: Duration::ZERO, + ..ReplicationConfig::default() + }), + ..TestNetworkConfig::minimal() + } +} + +/// The node, out of five, that is outside the retention width for `record`. +async fn out_of_range_node(harness: &TestHarness, record: &Pointer) -> usize { + let view = harness.test_node(0).expect("node"); + let p2p = view.p2p_node.as_ref().expect("p2p"); + let inside: Vec = p2p + .dht_manager() + .find_closest_nodes_local_with_self(&record.address(), 4) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + (0..harness.node_count()) + .find(|i| { + harness + .test_node(*i) + .is_some_and(|node| !inside.contains(&peer(node))) + }) + .expect("one node is outside a width of four in five") +} + +/// Pruning deletes a record the node is no longer responsible for once the +/// current close group prove they hold it, and keeps it while they do not. +#[tokio::test] +#[serial] +async fn pruning_deletes_only_once_the_close_group_proves_it_holds_the_record() { + // A paid width no five-node network can complete, so deletion needs the + // close group's proof rather than taking the far-away fast path. + let harness = TestHarness::setup_with_config(prune_network(20)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + store(harness.test_node(pruner).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + // Nobody else holds it yet: nothing proves it is safe to drop. + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &[pruner]).await; + let pruner_node = harness.test_node(pruner).expect("node"); + let pruning = replication(pruner_node); + pruning + .prune_pass(true, Some(commitments(pruner_node))) + .await; + assert!( + holds(harness.test_node(pruner).expect("node"), &record), + "the only copy was pruned" + ); + + // Once the rest hold it, the close group proves it and the pruner drops it. + for i in others(&harness, &[pruner]) { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + pruning + .prune_pass(true, Some(commitments(pruner_node))) + .await; + assert!( + held(harness.test_node(pruner).expect("node"), &record).is_none(), + "the record was not pruned though the close group holds it" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node outside even a complete paid-width group drops the record without +/// asking anyone, as a chunk is. +#[tokio::test] +#[serial] +async fn a_node_far_outside_the_group_prunes_without_asking() { + let harness = TestHarness::setup_with_config(prune_network(3)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + store(harness.test_node(pruner).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + let pruner_node = harness.test_node(pruner).expect("node"); + replication(pruner_node) + .prune_pass(false, Some(commitments(pruner_node))) + .await; + assert!( + held(pruner_node, &record).is_none(), + "a far-away record was kept" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A pointer a retained storage commitment still holds is never pruned, as a +/// chunk is not: a peer pinning that commitment may yet audit it, and a node +/// that had deleted it would fail. Once no retained commitment holds it, it +/// goes. +#[tokio::test] +#[serial] +async fn pruning_keeps_a_pointer_a_retained_commitment_still_holds() { + let harness = TestHarness::setup_with_config(prune_network(3)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + let pruner_node = harness.test_node(pruner).expect("node"); + store(pruner_node) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + // The commitment the node gossiped while it was still responsible. + let (node_pk, node_sk) = owner(); + let committed = BuiltCommitment::build( + vec![(record.address(), pointer_leaf_hash(&record.address()))], + peer(pruner_node).as_bytes(), + &node_sk, + &node_pk.to_bytes(), + ) + .expect("commitment"); + let state = commitments(pruner_node); + state.rotate(committed); + + replication(pruner_node) + .prune_pass(false, Some(state)) + .await; + assert!( + holds(pruner_node, &record), + "a pointer a retained commitment holds was pruned" + ); + + state.clear_all(); + replication(pruner_node) + .prune_pass(false, Some(state)) + .await; + assert!( + held(pruner_node, &record).is_none(), + "the pointer was kept after no commitment held it" + ); + + harness.teardown().await.expect("teardown"); +} + +/// Store `count` pointers on node `holder`, have it commit to them, and hand +/// that commitment to node `auditor`, as its gossip would. Returns what the +/// holder committed to. +async fn commit_pointers( + harness: &TestHarness, + holder: usize, + auditor: usize, + count: usize, +) -> Vec { + let holder_node = harness.test_node(holder).expect("holder"); + let records: Vec = (0..count) + .map(|_| { + let (pk, sk) = owner(); + signed(&pk, &sk, 1, 1) + }) + .collect(); + for record in &records { + store(holder_node) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + + let engine = holder_node.replication_engine.as_ref().expect("engine"); + engine.rebuild_commitment_now().await.expect("rebuild"); + let committed = engine + .commitment_state() + .current() + .expect("a current commitment"); + let pointers = committed.pointer_leaf_keys(); + assert!( + !pointers.is_empty(), + "the holder committed to none of the pointers it is responsible for" + ); + assert_eq!( + committed.leaf_keys(), + pointers, + "the holder has no chunks, so every leaf audited is a pointer" + ); + + harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .inject_peer_commitment_for_test(&peer(holder_node), committed.commitment().clone()) + .await; + records + .into_iter() + .filter(|record| pointers.contains(&record.address())) + .collect() +} + +/// A node holding the pointers it committed to passes the storage audit over +/// the wire, proving each opened one with its signed record. +#[tokio::test] +#[serial] +async fn a_node_holding_its_committed_pointers_passes_the_storage_audit() { + let harness = TestHarness::setup_small().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + let (holder, auditor) = (3, 4); + commit_pointers(&harness, holder, auditor, 48).await; + + let holder_peer = peer(harness.test_node(holder).expect("holder")); + let result = harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .audit_peer_now(&holder_peer) + .await; + assert!( + matches!(result, AuditTickResult::Passed { keys_checked, .. } if keys_checked >= 1), + "an honest pointer holder must pass, got {result:?}" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node that dropped the pointers it committed to fails the storage audit, +/// exactly as a node that dropped its chunks does. +#[tokio::test] +#[serial] +async fn a_node_that_dropped_its_committed_pointers_fails_the_storage_audit() { + let harness = TestHarness::setup_small().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + let (holder, auditor) = (5, 6); + let committed = commit_pointers(&harness, holder, auditor, 48).await; + + let holder_node = harness.test_node(holder).expect("holder"); + for record in &committed { + assert!(store(holder_node) + .delete(&record.address()) + .await + .expect("delete")); + } + + let result = harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .audit_peer_now(&peer(holder_node)) + .await; + assert!( + matches!(result, AuditTickResult::Failed { .. }), + "a node that dropped its committed pointers must fail, got {result:?}" + ); + + harness.teardown().await.expect("teardown"); +} diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index a082efd0..6e878709 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -236,6 +236,20 @@ pub struct TestNetworkConfig { pub storage_disk_reserve_overrides: HashMap, } +impl TestNetworkConfig { + /// The same configuration on a freshly drawn port range and data + /// directory, for a network whose first bring-up could not bind. + #[must_use] + pub fn with_fresh_ports(self) -> Self { + let fresh = Self::default(); + Self { + base_port: fresh.base_port, + test_data_dir: fresh.test_data_dir, + ..self + } + } +} + impl Default for TestNetworkConfig { fn default() -> Self { let mut rng = rand::thread_rng(); @@ -1180,11 +1194,20 @@ impl TestNetwork { .map_or_else(|_| vec![], |sig| sig.as_bytes().to_vec()) }); - Ok(AntProtocol::new( - Arc::new(storage), - Arc::new(payment_verifier), - Arc::new(quote_generator), - )) + // Pointers beside the chunks, wired as a real node wires them (ADR-0016). + let storage = Arc::new(storage); + let payment_verifier = Arc::new(payment_verifier); + let pointer_store = ant_node::pointer::PointerStore::new(data_dir) + .await + .map_err(|e| TestnetError::Core(format!("Failed to create pointer store: {e}")))?; + let pointers = ant_node::pointer::PointerService::new(pointer_store) + .with_chunk_store(Arc::clone(&storage)) + .with_payments(Arc::clone(&payment_verifier)); + + Ok( + AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers), + ) } /// Start a single node. @@ -1348,6 +1371,12 @@ impl TestNetwork { .await { Ok(mut engine) => { + // Pointers replicate through the same engine (ADR-0016). + if let Some(service) = protocol.pointer_service() { + let (writes, fresh_writes) = tokio::sync::mpsc::unbounded_channel(); + service.attach_fresh_writes(writes); + engine.with_pointers(service.store().clone(), fresh_writes); + } let dht_events = p2p.dht_manager().subscribe_events(); engine.start(dht_events); node.replication_engine = Some(engine); diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 5f970a08..9200e62b 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -614,7 +614,8 @@ async fn slice_challenge_opens_valid_blocks_for_committed_keys() { "nonced opening must fold to the honest nonced root" ); } - other @ SubtreeSliceItem::Absent { .. } => { + other @ (SubtreeSliceItem::Absent { .. } + | SubtreeSliceItem::PointerRecord { .. }) => { panic!("expected Present for stored committed key, got {other:?}") } } @@ -703,7 +704,8 @@ async fn slice_challenge_coalesces_duplicate_and_interleaved_openings() { ); seen.push((*key, *block_index)); } - other @ SubtreeSliceItem::Absent { .. } => { + other @ (SubtreeSliceItem::Absent { .. } + | SubtreeSliceItem::PointerRecord { .. }) => { panic!("expected Present for a stored committed key, got {other:?}") } } diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs new file mode 100644 index 00000000..4e1864aa --- /dev/null +++ b/tests/pointer_convergence.rs @@ -0,0 +1,665 @@ +//! Convergence and payment-binding properties of the pointer merge rule. +//! +//! ADR-0016's claim is that selecting the maximum over a total order on +//! *states* is idempotent, commutative and associative, so every node reaches +//! the same value from any delivery interleaving given the same record set. +//! These are the property tests behind that claim, plus the two anti-abuse +//! properties the merge rule exists to provide: one payment funds one state, +//! and no re-signature of a stored state can displace it. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::BTreeSet; + +use ant_protocol::pointer::{Pointer, PointerTarget, PointerTargetKind}; +use proptest::prelude::*; +use saorsa_pqc::api::sig::{ + ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey, MlDsaSignature, MlDsaVariant, +}; + +/// A deterministic keypair, so a failing case is reproducible from its seed. +fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) +} + +fn signed(seed: u8, counter: u64, target_byte: u8, kind: PointerTargetKind) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(kind, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("signing a pointer") +} + +/// Fold a delivery order down to the winner, as a node's store does. +fn winner<'a>(order: &[&'a Pointer]) -> &'a Pointer { + let mut best = order.first().copied().expect("non-empty delivery"); + for candidate in order.iter().skip(1) { + if candidate.replaces(best) { + best = candidate; + } + } + best +} + +/// Visit every permutation of `order`, calling `check` on each. +fn permute( + order: &mut Vec<&Pointer>, + start: usize, + check: &mut impl FnMut(&[&Pointer]) -> Result<(), TestCaseError>, + seen: &mut usize, +) -> Result<(), TestCaseError> { + if start == order.len() { + *seen += 1; + return check(order); + } + for i in start..order.len() { + order.swap(start, i); + permute(order, start + 1, check, seen)?; + order.swap(start, i); + } + Ok(()) +} + +/// `n!`, for asserting that every permutation really was visited. +fn factorial(n: usize) -> usize { + (1..=n).product() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(24))] + + /// Any permutation of a record set reaches the same state. + #[test] + fn every_delivery_order_converges( + counters in prop::collection::vec(0u64..4, 2..6), + targets in prop::collection::vec(0u8..4, 2..6), + ) { + let len = counters.len().min(targets.len()); + let records: Vec = (0..len) + .map(|i| { + let counter = counters.get(i).copied().unwrap_or(0); + let target = targets.get(i).copied().unwrap_or(0); + signed(1, counter, target, PointerTargetKind::Chunk) + }) + .collect(); + + let forward: Vec<&Pointer> = records.iter().collect(); + let expected = winner(&forward).state_id(); + + // Every permutation, not a sample of them: the sets are small enough + // that "any delivery order" can be checked exhaustively. + let mut order: Vec<&Pointer> = records.iter().collect(); + let mut permutations = 0usize; + permute(&mut order, 0, &mut |candidate| { + prop_assert_eq!(winner(candidate).state_id(), expected); + Ok(()) + }, &mut permutations)?; + prop_assert_eq!(permutations, factorial(records.len())); + } + + /// Delivering a record twice changes nothing: the fold is idempotent. + #[test] + fn duplicate_delivery_changes_nothing( + counter in 0u64..8, + target in 0u8..8, + extra_counter in 0u64..8, + extra_target in 0u8..8, + ) { + let first = signed(1, counter, target, PointerTargetKind::Chunk); + let second = signed(1, extra_counter, extra_target, PointerTargetKind::Chunk); + + let once = winner(&[&first, &second]).state_id(); + let twice = winner(&[&first, &second, &first, &second]).state_id(); + let thrice = winner(&[&second, &first, &second, &first, &first]).state_id(); + prop_assert_eq!(once, twice); + prop_assert_eq!(once, thrice); + } + + /// Exactly one of `a replaces b` / `b replaces a` holds for distinct states, + /// and neither holds for equal ones. Without this two nodes could disagree. + #[test] + fn the_comparator_is_antisymmetric( + left_counter in 0u64..4, + left_target in 0u8..4, + right_counter in 0u64..4, + right_target in 0u8..4, + ) { + let left = signed(1, left_counter, left_target, PointerTargetKind::Chunk); + let right = signed(1, right_counter, right_target, PointerTargetKind::Chunk); + + if left.state_id() == right.state_id() { + prop_assert!(!left.replaces(&right)); + prop_assert!(!right.replaces(&left)); + } else { + prop_assert_ne!(left.replaces(&right), right.replaces(&left)); + } + } + + /// No number of re-signatures of one state can displace it, and all of them + /// pay under a single identifier. + #[test] + fn re_signing_one_state_neither_wins_nor_pays_again( + counter in 0u64..64, + target in 0u8..64, + variants in 2usize..12, + ) { + let records: Vec = (0..variants) + .map(|_| signed(1, counter, target, PointerTargetKind::Chunk)) + .collect(); + + let encodings: BTreeSet> = records.iter().map(Pointer::to_bytes).collect(); + prop_assert_eq!(encodings.len(), variants, "ML-DSA signing is randomized"); + + let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); + prop_assert_eq!(states.len(), 1, "one state, so one paid identifier"); + + for a in &records { + for b in &records { + prop_assert!(!a.replaces(b)); + } + } + } + + /// A change to any signed field changes the paid identifier, so a receipt + /// bought for one update cannot fund another. (`state_id` is a 256-bit hash + /// over a 1,994-byte body, so this is collision resistance, not injectivity; + /// what is testable is that each field actually reaches the hash.) + #[test] + fn every_signed_field_changes_the_paid_identifier( + counter in 0u64..u64::MAX, + tag in any::(), + target in any::<[u8; 32]>(), + ) { + let (pk, sk) = keypair(1); + let base = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, target)) + .expect("sign"); + + let other_counter = Pointer::sign( + &sk, &pk, counter.wrapping_add(1), PointerTarget::from_raw_tag(tag, target), + ).expect("sign"); + let other_tag = Pointer::sign( + &sk, &pk, counter, PointerTarget::from_raw_tag(tag.wrapping_add(1), target), + ).expect("sign"); + let mut flipped = target; + flipped[0] ^= 1; + let other_target = Pointer::sign( + &sk, &pk, counter, PointerTarget::from_raw_tag(tag, flipped), + ).expect("sign"); + let (other_pk, other_sk) = keypair(2); + let other_owner = Pointer::sign( + &other_sk, &other_pk, counter, PointerTarget::from_raw_tag(tag, target), + ).expect("sign"); + + let ids: BTreeSet<_> = [&base, &other_counter, &other_tag, &other_target, &other_owner] + .iter() + .map(|record| record.state_id()) + .collect(); + prop_assert_eq!(ids.len(), 5, "counter, tag, target and owner all reach state_id"); + } + + /// Every record of one owner lives at one address, whatever it says — + /// including under a target tag this build does not know. + #[test] + fn the_address_tracks_only_the_owner( + counter in 0u64..1000, + tag in any::(), + target_bytes in any::<[u8; 32]>(), + ) { + let (pk, sk) = keypair(3); + let target = PointerTarget::from_raw_tag(tag, target_bytes); + let record = Pointer::sign(&sk, &pk, counter, target).expect("sign"); + let reference = signed(3, 0, 0, PointerTargetKind::Chunk); + prop_assert_eq!(record.address(), reference.address()); + prop_assert_ne!(record.address(), signed(4, 0, 0, PointerTargetKind::Chunk).address()); + + // An unknown tag round-trips untouched; the node never interprets it. + let parsed = Pointer::from_bytes(&record.to_bytes()).expect("parse"); + prop_assert_eq!(parsed.target().kind_tag(), tag); + prop_assert_eq!(parsed.target().address, target_bytes); + } + + /// The merge order holds over arbitrary target bytes, not just the + /// repeated-byte targets the other cases use. + #[test] + fn arbitrary_targets_order_by_their_bytes( + counter in 0u64..4, + left in any::<[u8; 32]>(), + right in any::<[u8; 32]>(), + tag in any::(), + ) { + let (pk, sk) = keypair(6); + let a = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, left)) + .expect("sign"); + let b = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, right)) + .expect("sign"); + + match left.cmp(&right) { + std::cmp::Ordering::Less => { + prop_assert!(a.replaces(&b), "smaller target bytes win"); + prop_assert!(!b.replaces(&a)); + } + std::cmp::Ordering::Greater => { + prop_assert!(b.replaces(&a)); + prop_assert!(!a.replaces(&b)); + } + std::cmp::Ordering::Equal => { + prop_assert!(!a.replaces(&b)); + prop_assert!(!b.replaces(&a)); + } + } + } + + /// Any record that parses is correctly signed; any mutation of its bytes + /// either fails to parse or fails to verify. Nothing forged gets through. + #[test] + fn no_mutation_of_a_record_survives_validation( + index in 0usize..5303, + mask in 1u8..255, + ) { + let record = signed(5, 11, 22, PointerTargetKind::Chunk); + let mut bytes = record.to_bytes(); + let Some(byte) = bytes.get_mut(index) else { + return Ok(()); + }; + *byte ^= mask; + + match Pointer::from_bytes(&bytes) { + Err(_) => {} + Ok(parsed) => { + // The only mutations that can verify are ones that produced a + // different but still-valid signature encoding of the same + // body, which cannot happen for a single flipped byte. If one + // ever does, it must still be the same state. + prop_assert_eq!(parsed.state_id(), record.state_id()); + } + } + } +} + +/// A worked instance of the attack the merge rule exists to stop: 64 valid +/// signatures over one paid state, submitted worst-first. +#[test] +fn sixty_four_signatures_over_one_state_yield_one_winner() { + let records: Vec = (0..64) + .map(|_| signed(7, 9, 9, PointerTargetKind::Chunk)) + .collect(); + + let encodings: BTreeSet> = records.iter().map(Pointer::to_bytes).collect(); + assert_eq!(encodings.len(), 64, "64 distinct valid encodings"); + + let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); + assert_eq!(states.len(), 1, "all 64 sit at one paid identifier"); + + // Sorted worst-first is the submission order that would have made every + // record win under a byte-ordering tie-break. + let mut sorted: Vec<&Pointer> = records.iter().collect(); + sorted.sort_by_key(|record| record.to_bytes()); + + let first = sorted.first().copied().expect("non-empty"); + for candidate in &sorted { + assert!( + !candidate.replaces(first), + "no re-signature of a stored state may displace it" + ); + } + assert_eq!(winner(&sorted).state_id(), first.state_id()); +} + +/// Golden vectors pinning the wire format. +/// +/// Field offsets, the big-endian counter, the inclusion of every owner byte and +/// all 33 target bytes, and the two domain separators are consensus: a node that +/// disagrees about any of them computes a different address or a different paid +/// identifier and silently partitions. These assertions restate the format +/// independently of the code under test, so a refactor that changes it fails +/// here rather than in production. +#[test] +fn the_wire_format_is_what_the_adr_says() { + use blake3::Hasher; + + let (pk, sk) = keypair(42); + let target_address = [0xABu8; 32]; + let counter = 0x0102_0304_0506_0708u64; + let record = Pointer::sign( + &sk, + &pk, + counter, + PointerTarget::from_raw_tag(0x5A, target_address), + ) + .expect("sign"); + let bytes = record.to_bytes(); + + // Layout. + assert_eq!(bytes.len(), 5303, "1 + 1952 + 8 + 33 + 3309"); + assert_eq!(bytes.first().copied(), Some(1u8), "format_version is 1"); + assert_eq!( + bytes.get(1..1953).expect("owner range"), + pk.to_bytes().as_slice(), + "every owner byte is carried verbatim at offset 1" + ); + assert_eq!( + bytes.get(1953..1961).expect("counter range"), + &[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08], + "the counter is big-endian at offset 1953" + ); + assert_eq!( + bytes.get(1961).copied(), + Some(0x5A), + "the target tag sits at offset 1961, uninterpreted" + ); + assert_eq!( + bytes.get(1962..1994).expect("target range"), + target_address.as_slice(), + "all 32 target address bytes follow the tag" + ); + + // Fixed expected values for the fixed seed. These are consensus: a build + // that computes anything else partitions from the network, so they are + // written out rather than recomputed from the code under test. + assert_eq!( + hex::encode(record.address()), + "83671eca6ee18b38987207922fdc23a1c378818c8a076b69dcbd40509f8786a8", + "the pointer address for seed 42" + ); + assert_eq!( + hex::encode(record.state_id()), + "2d611afab3f72642b0196d99faebb74fb9b4f659f0f0b3b4f73dc85847efab42", + "the paid state identifier for this record" + ); + + // And the derivations those constants come from, restated independently. + assert_eq!( + record.address(), + blake3::derive_key("autonomi.pointer.address.v1", &pk.to_bytes()), + "the address derives from the owner key alone" + ); + assert_eq!( + record.state_id(), + blake3::derive_key( + "autonomi.pointer.state.v1", + bytes.get(..1994).expect("body range") + ), + "state_id covers the whole body and nothing else" + ); + + // Derive-key, not a hash of a prefix: a chunk is addressed by BLAKE3 over + // its content, so a prefix construction would put both identities inside + // the chunk address space for anyone who could write the preimage. + let mut prefixed = Hasher::new(); + prefixed.update(b"autonomi.pointer.address.v1"); + prefixed.update(&pk.to_bytes()); + assert_ne!( + record.address(), + *prefixed.finalize().as_bytes(), + "the prefix construction must not still produce the address" + ); + + // The signature verifies over the body under the literal context, and does + // not verify under a different one. The context is consensus too. + let dsa = ml_dsa_65(); + let signature = MlDsaSignature::from_bytes( + MlDsaVariant::MlDsa65, + bytes.get(1994..).expect("signature range"), + ) + .expect("signature parses"); + assert!( + dsa.verify_with_context( + &pk, + bytes.get(..1994).expect("body range"), + &signature, + b"autonomi.pointer.head.v1", + ) + .expect("verify"), + "the signature is over the body under autonomi.pointer.head.v1" + ); + assert!( + !dsa.verify_with_context( + &pk, + bytes.get(..1994).expect("body range"), + &signature, + b"autonomi.pointer.head.v2", + ) + .expect("verify"), + "and not under any other context" + ); + + // The domains are distinct, so neither identifier can be mistaken for the + // other or for a plain content address. + assert_ne!(record.address(), record.state_id()); + assert_ne!( + record.address(), + *blake3::hash(&pk.to_bytes()).as_bytes(), + "the address is domain-separated from a bare hash of the key" + ); +} + +/// The same attack, driven through a real store: 64 valid signatures over one +/// paid state must produce exactly one file and never rewrite it. +/// +/// The fold above proves the comparator refuses them; this proves the storage +/// layer does, which is where the cost would actually have been paid. +#[tokio::test] +async fn sixty_four_signatures_buy_exactly_one_write() { + use ant_node::pointer::store::{PointerStore, PutOutcome}; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("open store"); + + let first = signed(7, 9, 9, PointerTargetKind::Chunk); + assert_eq!( + store.put_bytes(&first.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + + let path = store.file_for(&first.address()); + let stored = std::fs::read(&path).expect("read back"); + + for _ in 0..63 { + let variant = signed(7, 9, 9, PointerTargetKind::Chunk); + assert_ne!( + variant.to_bytes(), + first.to_bytes(), + "signing is randomized" + ); + assert_eq!( + store.put_bytes(&variant.to_bytes()).await.expect("put"), + PutOutcome::Unchanged + ); + } + + assert_eq!( + std::fs::read(&path).expect("read back"), + stored, + "63 re-signatures after the first must not touch the stored bytes" + ); + assert_eq!(store.len(), 1, "one address, one record"); + + // Records live one level down, in shard directories. + let files: Vec<_> = std::fs::read_dir(store.dir()) + .expect("read dir") + .filter_map(std::result::Result::ok) + .filter(|shard| shard.path().is_dir()) + .flat_map(|shard| std::fs::read_dir(shard.path()).expect("read shard")) + .filter_map(std::result::Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|name| !name.starts_with('.')) + .collect(); + assert_eq!(files.len(), 1, "one payment bought one file: {files:?}"); +} + +// ============================================================================= +// Version and counter gaps +// ============================================================================= + +/// A version-only change must not reuse a paid state identifier. +/// +/// `format_version` is the first byte of the signed body and `state_id` hashes +/// the whole body, so a record differing only in its version is a different +/// state. Without this a future format could spend a receipt bought for +/// version 1 — the free-update defect, reintroduced through the version field. +#[test] +fn a_version_only_change_cannot_reuse_a_paid_state_identifier() { + use ant_protocol::pointer::{state_id_for_body, POINTER_BODY_LEN, POINTER_FORMAT_VERSION}; + + let record = signed(11, 9, 9, PointerTargetKind::Chunk); + let mut body = record + .to_bytes() + .get(..POINTER_BODY_LEN) + .expect("body") + .to_vec(); + + let paid = state_id_for_body(&body); + assert_eq!( + paid, + record.state_id(), + "the baseline is the record's own id" + ); + + let mut seen = BTreeSet::new(); + seen.insert(paid); + for version in 0u8..=u8::MAX { + if version == POINTER_FORMAT_VERSION { + continue; + } + if let Some(byte) = body.first_mut() { + *byte = version; + } + let other = state_id_for_body(&body); + assert_ne!( + other, paid, + "version {version} must not share version {POINTER_FORMAT_VERSION}'s paid id" + ); + assert!( + seen.insert(other), + "version {version} collided with another version's paid id" + ); + } + assert_eq!(seen.len(), 256, "every version has its own paid identifier"); +} + +/// A record of an unknown version is refused outright, so it can never be +/// stored under version 1's authority even if someone paid for it. +#[test] +fn an_unknown_version_is_refused_rather_than_accepted_at_its_own_price() { + use ant_protocol::pointer::{PointerError, PointerState, POINTER_FORMAT_VERSION}; + + let record = signed(12, 1, 1, PointerTargetKind::Chunk); + for version in [0u8, 2, 7, u8::MAX] { + assert_ne!(version, POINTER_FORMAT_VERSION); + let mut bytes = record.to_bytes(); + if let Some(byte) = bytes.first_mut() { + *byte = version; + } + assert!(matches!( + Pointer::from_bytes(&bytes), + Err(PointerError::UnknownFormatVersion(v)) if v == version + )); + assert!(matches!( + PointerState::parse(&bytes), + Err(PointerError::UnknownFormatVersion(v)) if v == version + )); + } +} + +/// At `u64::MAX` no *counter* can out-rank the winner, but a smaller *target* +/// still can. That asymmetry is why migration has to happen before the terminal +/// update rather than on it. +#[test] +fn a_terminal_counter_cannot_be_out_counted_only_out_targeted() { + let terminal = signed(13, u64::MAX, 5, PointerTargetKind::Chunk); + assert!(terminal.is_terminal()); + assert!(terminal.next_counter().is_err(), "no successor exists"); + + // Nothing at any lower counter replaces it. + for counter in [0u64, 1, 1000, u64::MAX - 2, u64::MAX - 1] { + let earlier = signed(13, counter, 0, PointerTargetKind::Chunk); + assert!(!earlier.replaces(&terminal)); + assert!(terminal.replaces(&earlier)); + } + + // The order does not degenerate there: equal-counter conflicts at the + // maximum still resolve deterministically, so replicas cannot split. + let low = signed(13, u64::MAX, 1, PointerTargetKind::Chunk); + let high = signed(13, u64::MAX, 2, PointerTargetKind::Chunk); + assert!(low.replaces(&high)); + assert!(!high.replaces(&low)); + assert_eq!(winner(&[&high, &low]).state_id(), low.state_id()); + assert_eq!(winner(&[&low, &high]).state_id(), low.state_id()); +} + +/// Why migration must happen *before* the terminal update. +/// +/// At `u64::MAX` the counter can no longer advance, but the pointer is not +/// frozen: the merge order still resolves equal counters by target bytes, and +/// *smaller* target bytes win. So a migration written at the terminal counter +/// can still be displaced — by the owner, or by anyone replaying an older +/// signed record of theirs with a smaller target. The only safe migration is +/// one made while a successor counter still exists, because a strictly larger +/// counter is the one move nothing can answer. +#[tokio::test] +async fn migration_must_happen_before_the_terminal_update() { + use ant_node::pointer::store::{PointerStore, PutOutcome}; + use ant_protocol::pointer::PointerTarget; + use saorsa_pqc::api::sig::ml_dsa_65; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + + let (pk, sk) = ml_dsa_65().generate_keypair_from_seed(&[14u8; 32]); + let sign_at = |counter: u64, target: PointerTarget| { + Pointer::sign(&sk, &pk, counter, target).expect("sign") + }; + + // One update short of the end: a successor counter still exists. + let penultimate = sign_at( + u64::MAX - 1, + PointerTarget::new(PointerTargetKind::Chunk, [0x10u8; 32]), + ); + assert!(!penultimate.is_terminal()); + assert_eq!( + store.put_bytes(&penultimate.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + + // The safe migration: spend the last counter. A strictly larger counter + // beats every target, so nothing at u64::MAX - 1 can answer it. + let migration = sign_at( + penultimate.next_counter().expect("successor exists"), + PointerTarget::new(PointerTargetKind::Pointer, [0x80u8; 32]), + ); + assert_eq!( + store.put_bytes(&migration.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert!(migration.is_terminal()); + assert!(migration.next_counter().is_err()); + + // Now the danger. The counter is spent, so the only remaining moves are to + // strictly smaller target bytes — and they still win. + let smaller_target = sign_at( + u64::MAX, + PointerTarget::new(PointerTargetKind::Chunk, [0x01u8; 32]), + ); + assert!( + smaller_target.to_bytes() != migration.to_bytes(), + "a genuinely different state" + ); + assert_eq!( + store + .put_bytes(&smaller_target.to_bytes()) + .await + .expect("put"), + PutOutcome::Changed, + "a terminal pointer is NOT frozen: a smaller target still displaces it" + ); + + // Larger target bytes cannot claw it back: the move is one-way. + assert_eq!( + store.put_bytes(&migration.to_bytes()).await.expect("put"), + PutOutcome::Stale, + "and the displaced migration can never be restored" + ); + + // Which is the whole point: a migration made at the terminal counter is + // not final, so it has to be made earlier, where the counter still answers. + assert!(smaller_target.replaces(&migration)); + assert!(!migration.replaces(&smaller_target)); +}