From e5c7a5cd4ae9f7d005b78b4a15f66731224f13b1 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 5 Aug 2026 18:25:44 -0600 Subject: [PATCH 01/14] Retry infra startup; guard truncated REST responses - Retry marketplace connect and cirdl circuit download (10x10s) - Abort REST retrieves with short bodies loudly instead of silently truncating --- archivist/archivist.nim | 52 +++++++++++++++++++++++-------------- archivist/conf.nim | 18 +++++++++++++ archivist/rest/api.nim | 15 ++++++++++- docker/docker-entrypoint.sh | 15 ++++++++--- 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/archivist/archivist.nim b/archivist/archivist.nim index 0b38c2738..765ea5918 100644 --- a/archivist/archivist.nim +++ b/archivist/archivist.nim @@ -57,7 +57,7 @@ type NodePrivateKey* = libp2p.PrivateKey # alias -proc connectMarketplace(s: NodeServer) {.async.} = +proc connectMarketplace(s: NodeServer) {.async: (raises: [CancelledError]).} = let config = s.config if config.persistence: @@ -65,28 +65,40 @@ proc connectMarketplace(s: NodeServer) {.async.} = error "Persistence enabled, but no Ethereum private key was set" quit QuitFailure - let marketplaceResult = await MarketplaceNode.connect( - ethProviderUrl = config.ethProvider, - ethPrivateKeyFile = ethPrivateKeyFile, - datastore = s.repoStore.metaDs, - storage = MarketplaceStorage.new(s.archivistNode, s.repoStore), - options = MarketplaceOptions( - marketplaceAddress: config.marketplaceAddress, - maxPriorityFeePerGas: config.maxPriorityFeePerGas, - requestCacheSize: config.marketplaceRequestCacheSize, - validationEnabled: config.validator, - validationMaxSlots: some config.validatorMaxSlots, - validationGroups: config.validatorGroups, - validationGroupIndex: some config.validatorGroupIndex, - useSystemClock: config.useSystemClock, - ), - ) + # The RPC node may not be reachable on the first attempt (e.g. the geth + # endpoint is still propagating after a restart); retry before giving up. + var marketplace: ?MarketplaceNode + for attempt in 1 .. config.marketplaceConnectRetries: + without res =? ( + await MarketplaceNode.connect( + ethProviderUrl = config.ethProvider, + ethPrivateKeyFile = ethPrivateKeyFile, + datastore = s.repoStore.metaDs, + storage = MarketplaceStorage.new(s.archivistNode, s.repoStore), + options = MarketplaceOptions( + marketplaceAddress: config.marketplaceAddress, + maxPriorityFeePerGas: config.maxPriorityFeePerGas, + requestCacheSize: config.marketplaceRequestCacheSize, + validationEnabled: config.validator, + validationMaxSlots: some config.validatorMaxSlots, + validationGroups: config.validatorGroups, + validationGroupIndex: some config.validatorGroupIndex, + useSystemClock: config.useSystemClock, + ), + ) + ), err: + error "Unable to connect to marketplace", error = err.msg, attempt = attempt + if attempt < config.marketplaceConnectRetries: + await sleepAsync(config.marketplaceConnectRetryDelay) + continue + + marketplace = res.some + break - without marketplace =? marketplaceResult, err: - error "Unable to connect to marketplace", error = err.msg + without marketplaceNode =? marketplace: quit QuitFailure - s.archivistNode.marketplace = marketplace + s.archivistNode.marketplace = marketplaceNode proc start*(s: NodeServer) {.async.} = trace "Starting node", config = $s.config diff --git a/archivist/conf.nim b/archivist/conf.nim index 31b88aa33..40b6f600d 100644 --- a/archivist/conf.nim +++ b/archivist/conf.nim @@ -48,6 +48,10 @@ export DefaultQuotaBytes, DefaultOverlayTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas +const + DefaultMarketplaceConnectRetries* = 10 + DefaultMarketplaceConnectRetryDelay* = 10 + type ThreadCount* = range[0 .. 256] proc defaultDataDir*(): string = @@ -331,6 +335,20 @@ type name: "use-system-clock" .}: bool + marketplaceConnectRetries* {. + desc: "Number of marketplace connection retries before giving up", + defaultValue: DefaultMarketplaceConnectRetries, + defaultValueDesc: $DefaultMarketplaceConnectRetries, + name: "marketplace-connect-retries" + .}: int + + marketplaceConnectRetryDelay* {. + desc: "Delay in seconds between marketplace connection retries", + defaultValue: DefaultMarketplaceConnectRetryDelay.seconds, + defaultValueDesc: $DefaultMarketplaceConnectRetryDelay, + name: "marketplace-connect-retry-delay" + .}: Duration + validator* {. desc: "Enables validator, requires an Ethereum node", defaultValue: false, diff --git a/archivist/rest/api.nim b/archivist/rest/api.nim index 855bdc971..9d61c14fd 100644 --- a/archivist/rest/api.nim +++ b/archivist/rest/api.nim @@ -155,15 +155,28 @@ proc retrieveCid( bytes += buff.len await resp.send(addr buff[0], buff.len) + + if bytes != contentLength.int: + # The response would end short of the declared Content-Length. Abort + # without finishing so the client sees the truncation loudly instead of + # a clean short body. + error "Response body shorter than declared", + cid = cid, bytes, declared = contentLength.int + return + await resp.finish() archivist_api_downloads.inc() except CancelledError as exc: raise exc except LPStreamError as exc: warn "Error streaming blocks", exc = exc.msg - resp.status = Http500 if resp.isPending(): + resp.status = Http500 await resp.sendBody(exc.msg) + else: + # The response already started - abort so the client sees the truncation + # loudly instead of a clean short body. + error "Response aborted mid-stream", cid = cid, bytes, error = exc.msg finally: info "Sent bytes", cid = cid, bytes if not lpStream.isNil: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index b8a470b11..249acb286 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -153,13 +153,22 @@ if [[ "${ARCHIVIST_PROVER}" == "true" && -z "${SKIP_DOWNLOAD_CIRCUIT}" ]]; then export ARCHIVIST_CIRCUIT_DIR="${ARCHIVIST_DATA_DIR}/circuits" fi - # Download circuit + # Download circuit - retry until the RPC node is reachable. The dist-test + # harness treats a single failure as a pod crash. mkdir -p "${ARCHIVIST_CIRCUIT_DIR}" chmod 700 "${ARCHIVIST_CIRCUIT_DIR}" download="cirdl ${ARCHIVIST_CIRCUIT_DIR} ${ARCHIVIST_ETH_PROVIDER} ${ARCHIVIST_MARKETPLACE_ADDRESS}" echo "${download}" - eval "${download}" - [[ $? -ne 0 ]] && { echo "Failed to download circuit files"; exit 1; } + n=0 + until eval "${download}"; do + n=$((n + 1)) + if [ $n -ge 10 ]; then + echo "Failed to download circuit files" + exit 1 + fi + echo "Circuit download attempt $n failed, retrying in 10s..." + sleep 10 + done fi # Show From a9cb639c92ab8dbd2c812360f26f655636552257 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 6 Aug 2026 14:50:25 -0600 Subject: [PATCH 02/14] Quarantine recently queried peers in refresh selection The knowledge-refresh fallback always asked the same aggregate top-K; stable scores and deterministic tie order pinned the same peers forever, so a fresh DHT-discovered provider was never refreshed, never reported presence, and was never fetched from. Add a query quarantine that skips peers recently sent a want list, randomize equal-score ordering in the rankers, and exempt peers with unsent wants so newly queued blocks still reach them. Reuse the orphaned topPeersForCid helper for the known-haves branch and wake at the earliest quarantine expiry instead of busy-looping. --- archivist/blockexchange/engine/engine.nim | 56 ++++++--- .../blockexchange/engine/peerscoring.nim | 4 + archivist/blockexchange/peers/peercontext.nim | 18 +++ .../blockexchange/engine/testengine.nim | 116 ++++++++++++++++++ .../blockexchange/engine/testpeerscoring.nim | 16 +++ 5 files changed, 193 insertions(+), 17 deletions(-) diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index 945771e28..e9e38c5c0 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -210,9 +210,11 @@ proc refreshBlockKnowledge( return decision.wakeAfter of SendKind.Full: await self.sendBatchedWantList(peer, decision.wants.toSeq, full = true) + peer.markQueried() return Duration.none of SendKind.Delta: await self.sendBatchedWantList(peer, decision.wants.toSeq, full = false) + peer.markQueried() return Duration.none proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) = @@ -223,6 +225,16 @@ proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) = else: current = wake.some +proc hasUnsentWants(self: BlockExcEngine, peer: BlockExcPeerCtx): bool = + ## True when some wanted block has not been asked of this peer yet. + ## The query quarantine only suppresses re-asks of the same want list - + ## newly queued wants must still reach the peer (mirrors decideSend's + ## delta computation without mutating the circuit state). + for address in self.pendingBlocks.wantList: + if address notin peer.blocks and address notin peer.alreadySent: + return true + false + proc searchForNewPeers(self: BlockExcEngine, cid: Cid) = if self.lastDiscRequest + DiscoveryRateLimit < Moment.now(): archivist_block_exchange_discovery_requests.inc() @@ -235,20 +247,24 @@ proc evictPeer(self: BlockExcEngine, peer: PeerId) {.gcsafe, async: (raises: []) self.peers.remove(peer) proc topPeersForCid*(self: BlockExcEngine, cid: Cid, topK: int): seq[BlockExcPeerCtx] = - let candidates = self.peers.peersHave(cid) - if candidates.len == 0: - return @[] - let ranked = rankPeersByScore(candidates, cid) - if ranked.len == 0: - return @[] - ranked[0 ..< min(topK, ranked.len)] + let ranked = rankPeersByScore(self.peers.peersHave(cid), cid) + var top: seq[BlockExcPeerCtx] + for peer in ranked: + if top.len >= topK: + break + if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer): + top.add(peer) + top proc topPeersByAggregate*(self: BlockExcEngine, topK: int): seq[BlockExcPeerCtx] = - let allPeers = toSeq(self.peers.peers.values) - let ranked = rankPeersByAggregate(allPeers) - if ranked.len == 0: - return @[] - ranked[0 ..< min(topK, ranked.len)] + let ranked = rankPeersByAggregate(toSeq(self.peers.peers.values)) + var top: seq[BlockExcPeerCtx] + for peer in ranked: + if top.len >= topK: + break + if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer): + top.add(peer) + top proc failBlockRequest( self: BlockExcEngine, @@ -882,11 +898,7 @@ proc new*( candidates = self.peers.peersHave(cid) topPeers = if candidates.len > 0: - let ranked = rankPeersByScore(candidates, cid) - if ranked.len == 0: - @[] - else: - ranked[0 ..< min(DefaultWantHaveTopK, ranked.len)] + self.topPeersForCid(cid, DefaultWantHaveTopK) else: self.topPeersByAggregate(DefaultWantHaveTopK) @@ -899,6 +911,16 @@ proc new*( minWakeHint(minWake, fut.read()) except CatchableError: discard + else: + # Every candidate was queried recently - wake when the first quarantine + # expires instead of re-running selection in a tight loop. + let pool = + if candidates.len > 0: + candidates + else: + toSeq(self.peers.peers.values) + for peer in pool: + minWakeHint(minWake, peer.queryQuarantineLeft()) let discoveredPeer = selectKnownPeer(address) if not discoveredPeer.isNil: diff --git a/archivist/blockexchange/engine/peerscoring.nim b/archivist/blockexchange/engine/peerscoring.nim index 46ea8aafa..c44aae33c 100644 --- a/archivist/blockexchange/engine/peerscoring.nim +++ b/archivist/blockexchange/engine/peerscoring.nim @@ -38,6 +38,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC let s = effectiveScore(peer, cid, now) ranked.add((s, peer)) + # Randomize equal-score groups (stable sort preserves shuffle order) + Rng.instance.shuffle(ranked) ranked.sort( proc(a, b: auto): int = cmp(b[0], a[0]) @@ -46,6 +48,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC proc rankPeersByAggregate*(peers: seq[BlockExcPeerCtx]): seq[BlockExcPeerCtx] = var ranked = peers.mapIt((aggregateScore(it), it)) + # Randomize equal-score groups (stable sort preserves shuffle order) + Rng.instance.shuffle(ranked) ranked.sort( proc(a, b: auto): int = cmp(b[0], a[0]) diff --git a/archivist/blockexchange/peers/peercontext.nim b/archivist/blockexchange/peers/peercontext.nim index e541a426e..e6d86305b 100644 --- a/archivist/blockexchange/peers/peercontext.nim +++ b/archivist/blockexchange/peers/peercontext.nim @@ -72,6 +72,7 @@ type scores*: Table[Cid, PeerScore] # scores per dataset (cid) disconnected: Future[void] # completed when peer is removed from store + lastQueriedAt: Moment # when this peer was last asked about wanted blocks proc openWindow*(self: BlockExcPeerCtx, now = Moment.now()) = self.state = WantListState.SendAll @@ -149,6 +150,17 @@ proc markBlockAsNotSent*(self: BlockExcPeerCtx, address: BlockAddress) = ## a cancelled send never left the peer, so no cooldown applies. self.blocksSent.del(address) +proc markQueried*(self: BlockExcPeerCtx, now = Moment.now()) = + self.lastQueriedAt = now + +proc queryQuarantineLeft*( + self: BlockExcPeerCtx, + quarantine: Duration = DefaultWantHaveResendTimeout, + now = Moment.now(), +): ?Duration = + let left = (self.lastQueriedAt + quarantine) - now + if left.nanoseconds > 0: left.some else: Duration.none + proc clearSentIfStale*( self: BlockExcPeerCtx, address: BlockAddress, cooldown: Duration ): bool = @@ -201,6 +213,11 @@ proc ensureScoreFor*(self: BlockExcPeerCtx, cid: Cid): PeerScore = peerScore proc aggregateScore*(peer: BlockExcPeerCtx): float = + if peer.scores.len == 0: + # No evidence yet - baseline like the per-cid score path does, so a + # freshly discovered peer is not ranked below every known peer. + return ColdStartScore + var total = 0.0 for _, score in peer.scores: score.computeScore(peer.blocksRequested.len) @@ -211,6 +228,7 @@ proc aggregateScore*(peer: BlockExcPeerCtx): float = proc effectiveScore*(peer: BlockExcPeerCtx, cid: Cid, now: Moment): float = without score =? peer.scoreFor(cid): return ColdStartScore + score.computeScore(peer.blocksRequested.len) applyDecay(score.score, score.lastUpdated, peer.blocksRequested.len, now) diff --git a/tests/archivist/blockexchange/engine/testengine.nim b/tests/archivist/blockexchange/engine/testengine.nim index ca1fcb4dc..1b97a891f 100644 --- a/tests/archivist/blockexchange/engine/testengine.nim +++ b/tests/archivist/blockexchange/engine/testengine.nim @@ -269,6 +269,122 @@ asyncchecksuite "NetworkStore engine handlers": check archivist_block_exchange_want_have_entries_received.value() == beforeEntries + blocks.len.float64 + test "Should rotate refresh targets past the query quarantine": + # Regression: the refresh fallback always asked the same aggregate top-K; + # a fresh provider with zero deliveries was never asked, never reported + # presence, and was never fetched from. Quarantine makes selection rotate. + let + testPeerStore = PeerCtxStore.new() + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, network, testDiscovery, testAdvertiser, testPeerStore, pendingBlocks + ) + + var peerIds: seq[PeerId] + for i in 0 ..< 4: + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + id = PeerId.init(key.getPublicKey().tryGet()).tryGet() + peerIds.add(id) + let ctx = BlockExcPeerCtx.new(id) + testPeerStore.add(ctx) + if i < 3: + for _ in 0 ..< 2: + ctx.ensureScoreFor(blocks[0].cid).recordDelivery(1024, 10.0) + + # Round 1: the three delivery-scored peers take all top-K slots. + let first = testEngine.topPeersByAggregate(DefaultWantHaveTopK) + check first.len == 3 + for i in 0 ..< 3: + check peerIds[i] in first.mapIt(it.id) + check peerIds[3] notin first.mapIt(it.id) + + # Quarantine them - the zero-score peer rotates in. + for peer in first: + peer.markQueried() + let second = testEngine.topPeersByAggregate(DefaultWantHaveTopK) + check peerIds[3] in second.mapIt(it.id) + + # Quarantine everyone - nothing left to refresh. + for peer in second: + peer.markQueried() + check testEngine.topPeersByAggregate(DefaultWantHaveTopK).len == 0 + + test "Should randomize equal-score peers in the aggregate ranking": + # Equal-score peers (e.g. all at ColdStartScore) must not pin one top-K + # forever - a stable store order would starve the tail of the tie group. + let + testPeerStore = PeerCtxStore.new() + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, network, testDiscovery, testAdvertiser, testPeerStore, pendingBlocks + ) + + for _ in 0 ..< 4: + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + id = PeerId.init(key.getPublicKey().tryGet()).tryGet() + testPeerStore.add(BlockExcPeerCtx.new(id)) + + var distinctTops: HashSet[string] + for _ in 0 ..< 200: + distinctTops.incl($testEngine.topPeersByAggregate(2).mapIt(it.id)) + + check distinctTops.len > 1 # equal-score peers never pin one top-K + + test "Should still refresh quarantined peers when there are unsent wants": + # Regression pin: the query quarantine must not suppress WantHaves for + # addresses a peer has not been asked about yet. The connect-time Full + # send quarantines the only peer, and a later request for a new block + # must still reach it (decideSend would emit a Delta for the new wants). + var asked = newSeq[PeerId]() + + proc sendWantList( + id: PeerId, + addresses: seq[BlockAddress], + priority: int32 = 0, + cancel: bool = false, + wantType: WantType = WantType.WantHave, + full: bool = false, + sendDontHave: bool = false, + ): Future[?!void] {.async: (raises: [CancelledError]).} = + asked.add(id) + success() + + let + testPeerStore = PeerCtxStore.new() + testNetwork = + BlockExcNetwork(request: BlockExcRequest(sendWantList: sendWantList)) + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, testNetwork, testDiscovery, testAdvertiser, testPeerStore, + pendingBlocks, + ) + + # A fresh peer with no wants asked yet, quarantined. + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + freshId = PeerId.init(key.getPublicKey().tryGet()).tryGet() + let fresh = BlockExcPeerCtx.new(freshId) + testPeerStore.add(fresh) + fresh.markQueried() + + let pending = testEngine.requestDelivery(BlockAddress.init(blocks[0].cid)).tryGet() + await sleepAsync(200.millis) + check asked.len > 0 # unsent wants override the quarantine - WantHave sent + await pending.cancelAndWait() + expect CancelledError: + discard await pending + test "Should handle want list - `dont-have`": let wantList = makeWantList(blocks.mapIt(it.cid), sendDontHave = true) var diff --git a/tests/archivist/blockexchange/engine/testpeerscoring.nim b/tests/archivist/blockexchange/engine/testpeerscoring.nim index 2cfc79dd2..afc52e278 100644 --- a/tests/archivist/blockexchange/engine/testpeerscoring.nim +++ b/tests/archivist/blockexchange/engine/testpeerscoring.nim @@ -93,3 +93,19 @@ suite "scoredPeer selector": let ranked = rankPeersByScore(@[peer], testCid) check ranked.len == 1 check ranked[0] == peer + + test "Should baseline aggregate score at ColdStartScore for new peers": + # A peer with no score entries at all must not rank at 0.0. + check aggregateScore(makeCtx()) == ColdStartScore + + test "Should NOT count WantHave evidence in the aggregate score": + # A Have-reply creates a score entry (setPresence), but counting it + # would let a peer inflate its score by reporting Haves it never + # delivers. The entry contributes 0 until deliveries happen. + check aggregateScore(makeCtxWithScore()) == 0.0 + + test "Should rank delivered peers above the baseline in aggregate": + let busy = makeCtxWithScore() + busy.ensureScoreFor(testCid).recordDelivery(1024, 10.0) + busy.ensureScoreFor(testCid).recordDelivery(1024, 10.0) + check aggregateScore(busy) > ColdStartScore From 793b83b4b33ef6d63ccdf49df2c80812fd1fcc1e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 6 Aug 2026 17:26:43 -0600 Subject: [PATCH 03/14] Advertise without retry window; rework discovery backoff - Remove the 10-minute advertise retry window: the sparse retry is now steady-state, gated only by table health, so small networks keep replicating provider records to newly-arriving peers - Rework the discovery find-backoff to a (due, mult) tuple table with multiplier-derived intervals; prune entries once a cid leaves the want list, including tree cids of leaf wants - Make nodesDiscovered a method so tests can override it; export DefaultMinAdvertisePeers - Add steady-state advertiser retry test and backoff-recovery test --- archivist/blockexchange/engine/advertiser.nim | 16 +--- archivist/blockexchange/engine/discovery.nim | 56 +++++++++-- archivist/discovery.nim | 2 +- .../discovery/testdiscoveryengine.nim | 92 +++++++++++++++++++ .../blockexchange/engine/testadvertiser.nim | 24 +++++ tests/archivist/helpers/mockdiscovery.nim | 10 ++ 6 files changed, 180 insertions(+), 20 deletions(-) diff --git a/archivist/blockexchange/engine/advertiser.nim b/archivist/blockexchange/engine/advertiser.nim index a7db1bb20..8c60e2287 100644 --- a/archivist/blockexchange/engine/advertiser.nim +++ b/archivist/blockexchange/engine/advertiser.nim @@ -35,9 +35,8 @@ logScope: const DefaultConcurrentAdvertRequests = 10 DefaultAdvertiseLoopSleep = 30.minutes - DefaultMinAdvertisePeers = 16 + DefaultMinAdvertisePeers* = 16 DefaultAdvertiseRetrySleep = 30.seconds - DefaultAdvertiseRetryWindow = 10.minutes type Advertiser* = ref object of RootObj localStore*: BlockStore # Local block store for this instance @@ -55,8 +54,6 @@ type Advertiser* = ref object of RootObj pendingAdvRetries: Table[Cid, Future[void]] # Delayed sparse retry tasks minAdvertisePeers: int # Desired routing-table size before advertising advertiseRetrySleep: Duration # Delay before retrying sparse startup advertise - advertiseRetryWindow: Duration # Startup window for sparse advertise retries - startedAt: Moment # Time advertiser started proc addCidToQueue(b: Advertiser, cid: Cid) {.async: (raises: [CancelledError]).} = if cid notin b.advertiseQueue: @@ -113,18 +110,14 @@ proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} = info "Exiting advertise task loop" -proc hasElapsed(since: Moment, dur: Duration): bool = - (Moment.now() - since) >= dur - proc shouldRetryAdvertise(b: Advertiser): bool = - b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers and - not hasElapsed(b.startedAt, b.advertiseRetryWindow) + b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers proc delayedAdvertiseRetry(b: Advertiser, cid: Cid) {.async: (raises: []).} = try: await sleepAsync(b.advertiseRetrySleep) - if b.advertiserRunning and not hasElapsed(b.startedAt, b.advertiseRetryWindow): + if b.advertiserRunning and b.shouldRetryAdvertise(): trace "Requeueing advertisement retry", cid, nodes = b.discovery.nodesDiscovered(), target = b.minAdvertisePeers await b.addCidToQueue(cid) @@ -192,7 +185,6 @@ proc start*(b: Advertiser) {.async: (raises: []).} = doAssert(b.localStore.onBlockStored.isNone()) b.localStore.onBlockStored = onBlock.some - b.startedAt = Moment.now() for i in 0 ..< b.concurrentAdvReqs: let fut = b.processQueueLoop() b.trackedFutures.track(fut) @@ -225,7 +217,6 @@ proc new*( advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep, minAdvertisePeers = DefaultMinAdvertisePeers, advertiseRetrySleep = DefaultAdvertiseRetrySleep, - advertiseRetryWindow = DefaultAdvertiseRetryWindow, ): Advertiser = ## Create a advertiser instance ## @@ -240,5 +231,4 @@ proc new*( advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep, minAdvertisePeers: minAdvertisePeers, advertiseRetrySleep: advertiseRetrySleep, - advertiseRetryWindow: advertiseRetryWindow, ) diff --git a/archivist/blockexchange/engine/discovery.nim b/archivist/blockexchange/engine/discovery.nim index ad9f41956..99dcadab9 100644 --- a/archivist/blockexchange/engine/discovery.nim +++ b/archivist/blockexchange/engine/discovery.nim @@ -8,6 +8,7 @@ ## those terms. import std/sequtils +import std/sets import pkg/chronos import pkg/libp2p/cid @@ -37,6 +38,10 @@ const DefaultDiscoveryTimeout = 1.minutes DefaultMinPeersPerBlock = 3 DefaultDiscoveryLoopSleep = 3.seconds + DefaultDiscoverBackoffBase = 3.seconds + DefaultDiscoverBackoffMax = 60.seconds + DefaultDiscoverBackoffMaxMult = 5 + # base << 5 (96s) exceeds the 60s cap, so the interval saturates there type DiscoveryEngine* = ref object of RootObj localStore*: BlockStore # Local block store for this instance @@ -51,12 +56,24 @@ type DiscoveryEngine* = ref object of RootObj trackedFutures*: TrackedFutures # Tracked Discovery tasks futures minPeersPerBlock*: int # Max number of peers with block discoveryLoopSleep: Duration # Discovery loop sleep + discoverBackoffBase: Duration # Backoff base for failed discovery requests + discoverBackoffMax: Duration # Backoff cap for failed discovery requests inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]] # Inflight discovery requests + discoverBackoff: Table[Cid, tuple[due: Moment, mult: int]] + # Earliest retry time and backoff multiplier per CID proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} = try: while b.discEngineRunning: + # Drop backoff state for blocks that are no longer wanted - the table + # must not grow with every cid ever queued. wantListCids covers leaf + # wants' tree cids, which also get discovery finds (searchForNewPeers). + let wanted = toHashSet(toSeq(b.pendingBlocks.wantListCids)) + for cid in toSeq(b.discoverBackoff.keys): + if cid notin wanted: + b.discoverBackoff.del(cid) + for cid in toSeq(b.pendingBlocks.wantListBlockCids): await b.discoveryQueue.put(cid) @@ -64,6 +81,17 @@ proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} = except CancelledError: trace "Discovery loop cancelled" +proc nextDiscoverBackoff(b: DiscoveryEngine, cid: Cid) {.raises: [].} = + ## Advance the per-CID discovery backoff after a failed or empty find: + ## increment the stored multiplier (capped at + ## DefaultDiscoverBackoffMaxMult), then the next attempt is due in + ## base * 2^mult, capped at discoverBackoffMax. + var mult = 0 + b.discoverBackoff.withValue(cid, entry): + mult = min(entry[].mult + 1, DefaultDiscoverBackoffMaxMult) + let interval = min(b.discoverBackoffBase * (1 shl mult), b.discoverBackoffMax) + b.discoverBackoff[cid] = (due: Moment.now() + interval, mult: mult) + proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = ## Run discovery tasks ## @@ -77,8 +105,13 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = continue let haves = b.peers.peersHave(cid) - if haves.len < b.minPeersPerBlock: + var notDue = false + b.discoverBackoff.withValue(cid, entry): + notDue = Moment.now() < entry[].due + if notDue: + continue + let request = b.discovery.find(cid) b.inFlightDiscReqs[cid] = request archivist_inflight_discovery.set(b.inFlightDiscReqs.len.int64) @@ -89,11 +122,17 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = if (await request.withTimeout(DefaultDiscoveryTimeout)) and peers =? (await request).catch: - let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data))) - - for i, f in dialed: - if f.failed: - await b.discovery.removeProvider(peers[i].data.peerId) + if peers.len > 0: + b.discoverBackoff.del(cid) + + let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data))) + for i, f in dialed: + if f.failed: + await b.discovery.removeProvider(peers[i].data.peerId) + else: + b.nextDiscoverBackoff(cid) + else: + b.nextDiscoverBackoff(cid) except CancelledError: trace "Discovery task cancelled" return @@ -154,6 +193,8 @@ proc new*( concurrentDiscReqs = DefaultConcurrentDiscRequests, discoveryLoopSleep = DefaultDiscoveryLoopSleep, minPeersPerBlock = DefaultMinPeersPerBlock, + discoverBackoffBase = DefaultDiscoverBackoffBase, + discoverBackoffMax = DefaultDiscoverBackoffMax, ): DiscoveryEngine = ## Create a discovery engine instance for advertising services ## @@ -169,4 +210,7 @@ proc new*( inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](), discoveryLoopSleep: discoveryLoopSleep, minPeersPerBlock: minPeersPerBlock, + discoverBackoffBase: discoverBackoffBase, + discoverBackoffMax: discoverBackoffMax, + discoverBackoff: initTable[Cid, tuple[due: Moment, mult: int]](), ) diff --git a/archivist/discovery.nim b/archivist/discovery.nim index 0c609890a..c3f70363a 100644 --- a/archivist/discovery.nim +++ b/archivist/discovery.nim @@ -123,7 +123,7 @@ method provide*(d: Discovery, cid: Cid) {.async: (raises: [CancelledError]), bas except CatchableError as exc: warn "Error providing block", cid, exc = exc.msg -proc nodesDiscovered*(d: Discovery): int = +method nodesDiscovered*(d: Discovery): int {.base, gcsafe.} = if d.protocol.isNil: return 0 diff --git a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim index 91a5fe09e..9258144c8 100644 --- a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim @@ -220,3 +220,95 @@ asyncchecksuite "Test Discovery Engine": reqs.complete() await discoveryEngine.stop() + + test "Should back off failed discovery requests": + var + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + discoveryEngine = DiscoveryEngine.new( + localStore, + peerStore, + network, + blockDiscovery, + pendingBlocks, + discoveryLoopSleep = 100.millis, + ) + calls = 0 + + blockDiscovery.findBlockProvidersHandler = proc( + d: MockDiscovery, cid: Cid + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + calls.inc + # empty result is a failed discovery + + pendingBlocks.getPeerForBlock = alwaysRequeue(30.seconds) + await pendingBlocks.start() + discard pendingBlocks.getWantHandle(blocks[0].cid) + check toSeq(pendingBlocks.wantListBlockCids).len == 1 + + await discoveryEngine.start() + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 1 + + # Backoff (3s) has elapsed, the re-queued want is processed again + await sleepAsync(2.seconds + 300.millis) + check calls == 2 + + # Backoff doubles (3s -> 6s), so the third call must not happen yet + await sleepAsync(1.seconds) + check calls == 2 + + await discoveryEngine.stop() + await pendingBlocks.stop() + + test "Should rediscover immediately once a block leaves the want list": + var + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + discoveryEngine = DiscoveryEngine.new( + localStore, + peerStore, + network, + blockDiscovery, + pendingBlocks, + discoveryLoopSleep = 100.millis, + ) + calls = 0 + + blockDiscovery.findBlockProvidersHandler = proc( + d: MockDiscovery, cid: Cid + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + calls.inc + # empty result is a failed discovery + + pendingBlocks.getPeerForBlock = alwaysRequeue(30.seconds) + await pendingBlocks.start() + discard pendingBlocks.getWantHandle(blocks[0].cid) + await discoveryEngine.start() + + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 1 + + # Second failure grows the backoff to 6s + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(3.seconds + 300.millis) + check calls == 2 + + # Resolve the block - it leaves the want list; its backoff must be pruned + await pendingBlocks.resolve(BlockAddress.init(blocks[0].cid), blocks[0]) + await sleepAsync(500.millis) # prune runs every discoveryLoopSleep (100ms) + + # Re-request the same cid - discovery must fire without the stale 6s wait + discard pendingBlocks.getWantHandle(blocks[0].cid) + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 3 + + await discoveryEngine.stop() + await pendingBlocks.stop() diff --git a/tests/archivist/blockexchange/engine/testadvertiser.nim b/tests/archivist/blockexchange/engine/testadvertiser.nim index aac3344d2..3ddeae296 100644 --- a/tests/archivist/blockexchange/engine/testadvertiser.nim +++ b/tests/archivist/blockexchange/engine/testadvertiser.nim @@ -110,3 +110,27 @@ asyncchecksuite "Advertiser": check: localStore.onBlockStored.isNone() + + test "Should keep retrying while the routing table is below the minimum": + await advertiser.stop() + blockDiscovery.nodesDiscoveredHandler = proc(d: MockDiscovery): int = + 2 + advertiser = Advertiser.new( + localStore, + blockDiscovery, + minAdvertisePeers = 16, + advertiseRetrySleep = 500.millis, + ) + await advertiser.start() + + (await localStore.putBlock(manifestBlk)).tryGet() + + await sleepAsync(2500.millis) # several retry rounds at 500ms cadence + check advertised.len >= 6 # initial 2 (manifest + tree) + retries + + await sleepAsync(700.millis) # let any in-flight retry round drain + let count = advertised.len + blockDiscovery.nodesDiscoveredHandler = proc(d: MockDiscovery): int = + 16 + await sleepAsync(1500.millis) + check advertised.len == count # healthy table: retry cycle stops diff --git a/tests/archivist/helpers/mockdiscovery.nim b/tests/archivist/helpers/mockdiscovery.nim index 74ed64fd6..0ab0e0b72 100644 --- a/tests/archivist/helpers/mockdiscovery.nim +++ b/tests/archivist/helpers/mockdiscovery.nim @@ -11,9 +11,12 @@ import pkg/chronos import pkg/libp2p import pkg/questionable import pkg/archivist/discovery +import pkg/archivist/blockexchange/engine import pkg/contractabi/address as ca type MockDiscovery* = ref object of Discovery + nodesDiscoveredHandler*: proc(d: MockDiscovery): int {.gcsafe, raises: [].} + findBlockProvidersHandler*: proc( d: MockDiscovery, cid: Cid ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} @@ -32,6 +35,13 @@ type MockDiscovery* = ref object of Discovery proc new*(T: type MockDiscovery): MockDiscovery = MockDiscovery() +method nodesDiscovered*(d: MockDiscovery): int = + # Report a healthy table unless a test overrides it + if isNil(d.nodesDiscoveredHandler): + return DefaultMinAdvertisePeers + + d.nodesDiscoveredHandler(d) + proc findPeer*( d: Discovery, peerId: PeerId ): Future[?PeerRecord] {.async: (raises: [CancelledError]).} = From 748c356b68da571e3ba441531939ec03d563bd9e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sat, 8 Aug 2026 13:29:36 -0600 Subject: [PATCH 04/14] Expire repair overlays at the request end erasure.repair stored the reconstructed dataset under the original-tree overlay with no expiry, so putOverlay assigned the 30-day default TTL and the maintenance never cleaned it after the request ended (hosts retained the whole dataset - SlotCleanupTest 97 vs 33 blocks). Thread the request expiry from storeSlot through repair into the overlay writes, matching the flow's implicit cleanup design. --- archivist/erasure/erasure.nim | 20 ++++++++++++++++---- archivist/node.nim | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/archivist/erasure/erasure.nim b/archivist/erasure/erasure.nim index a1fe5808c..0bc137046 100644 --- a/archivist/erasure/erasure.nim +++ b/archivist/erasure/erasure.nim @@ -421,6 +421,7 @@ proc encode*( blocks: Natural, parity: Natural, strategy = SteppedStrategy, + expiry = ZeroSeconds, ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Encode a manifest into one that is erasure protected. ## @@ -474,6 +475,7 @@ proc encode*( ?await self.repoStore.withOverlay( manifest.treeCid, status = Repairing.some, + expiry = expiry, body = proc(): Future[?!Cid] {. closure, gcsafe, async: (raises: [CancelledError]) .} = @@ -526,7 +528,10 @@ proc leopardDecodeTask*( ctx[].result = ThreadSpawnRes[seq[seq[byte]]].ok(move recovered) proc decodeInternal( - self: Erasure, encodedTreeCid: Cid, targetCid: Cid, params: EncodingParams + self: Erasure, + encodedTreeCid: Cid, + targetCid: Cid, + params: EncodingParams, ): Future[?!(ref seq[Cid], seq[Natural])] {.async: (raises: [CancelledError]).} = logScope: encodedTreeCid = encodedTreeCid @@ -658,7 +663,7 @@ proc decode*( return decoded.success proc repair*( - self: Erasure, encoded: Manifest + self: Erasure, encoded: Manifest, expiry = ZeroSeconds ): Future[?!void] {.async: (raises: [CancelledError]).} = ## Repair a protected manifest by reconstructing the full dataset ## @@ -690,10 +695,13 @@ proc repair*( ?await self.repoStore.withOverlay( encoded.originalTreeCid, status = Repairing.some, + expiry = expiry, body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = let (cids, _) = - ?await self.decodeInternal(encoded.treeCid, encoded.originalTreeCid, params) + ?await self.decodeInternal( + encoded.treeCid, encoded.originalTreeCid, params + ) tree = # sync build: root comparison only (decode paths do not stream) ?ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]) @@ -714,7 +722,11 @@ proc repair*( # blocks for a valid slot - this is higly inneficient # we need to either fix leopard or use another implementation let repaired = - ?(await self.encode(encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy)) + ?( + await self.encode( + encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy, expiry + ) + ) trace "Successfully re-encoded original dataset" if repaired.treeCid != encoded.treeCid: diff --git a/archivist/node.nim b/archivist/node.nim index 2c4c2de89..75bdc9f3f 100644 --- a/archivist/node.nim +++ b/archivist/node.nim @@ -916,7 +916,7 @@ proc storeSlot*( self.networkStore, self.repoStore, leoEncoderProvider, leoDecoderProvider, self.taskpool, ) - if err =? (await erasure.repair(manifest)).errorOption: + if err =? (await erasure.repair(manifest, expiry)).errorOption: error "Unable to erasure decode repairing manifest", cid = manifest.treeCid, exc = err.msg return failure(err) From 72230a92f0edeef970318fbe4af033ddd963c44f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sat, 8 Aug 2026 22:03:36 -0600 Subject: [PATCH 05/14] Fix marketplace event-based reload and pastBlockTag underflow - Clamp pastBlockTag to genesis when the requested window exceeds the chain head: a fresh chain is shorter than the 30-day request window, and the UInt256 subtraction underflowed, failing every marketplace start with 'hex number > 64 bits' - Reload purchases and sales from onchain StorageRequested/SlotFilled event history instead of myRequests/mySlots - Guard errored-state restarts with the onchain request/slot state and signer check instead of the myRequests/mySlots membership list - Reserve slots only when the host token balance covers the collateral - Pin vendored contractabi with the readTail in-place decode fallback --- archivist/marketplace/abstractmarketplace.nim | 5 +++++ .../contracts/onchainmarketplace.nim | 9 ++++++++ archivist/marketplace/contracts/provider.nim | 9 +++++++- archivist/marketplace/purchasing.nim | 17 +++++++++++---- .../marketplace/purchasing/states/errored.nim | 5 ++++- archivist/marketplace/sales.nim | 9 ++++++-- .../marketplace/sales/states/errored.nim | 3 ++- .../sales/states/slotreserving.nim | 3 ++- tests/archivist/helpers/mockmarketplace.nim | 18 +++++++++++++--- .../marketplace/sales/states/testerrored.nim | 5 ++++- .../archivist/marketplace/testpurchasing.nim | 21 ++++++++++++------- vendor/nimble/contractabi | 2 +- 12 files changed, 83 insertions(+), 23 deletions(-) diff --git a/archivist/marketplace/abstractmarketplace.nim b/archivist/marketplace/abstractmarketplace.nim index 0d8744072..801a1cb25 100644 --- a/archivist/marketplace/abstractmarketplace.nim +++ b/archivist/marketplace/abstractmarketplace.nim @@ -230,6 +230,11 @@ method canReserveSlot*( ): Future[bool] {.base, async.} = raiseAssert("not implemented") +method tokensAvailable*( + marketplace: AbstractMarketplace +): Future[UInt256] {.base, async: (raises: [CancelledError, MarketplaceError]).} = + raiseAssert("not implemented") + method subscribeFulfillment*( marketplace: AbstractMarketplace, callback: OnFulfillment ): Future[Subscription] {.base, async.} = diff --git a/archivist/marketplace/contracts/onchainmarketplace.nim b/archivist/marketplace/contracts/onchainmarketplace.nim index 61276ec5a..140af0a47 100644 --- a/archivist/marketplace/contracts/onchainmarketplace.nim +++ b/archivist/marketplace/contracts/onchainmarketplace.nim @@ -366,6 +366,15 @@ method canReserveSlot*( convertEthersError("Unable to determine if slot can be reserved"): return await marketplace.contract.canReserveSlot(requestId, slotIndex) +method tokensAvailable*( + marketplace: OnChainMarketplace +): Future[UInt256] {.async: (raises: [CancelledError, MarketplaceError]).} = + convertEthersError("Failed to get token balance"): + let tokenAddress = await marketplace.contract.token() + let token = Erc20Token.new(tokenAddress, marketplace.signer) + let owner = await marketplace.signer.getAddress() + return await token.balanceOf(owner) + method subscribeRequests*( marketplace: OnChainMarketplace, callback: OnRequest ): Future[MarketSubscription] {.async.} = diff --git a/archivist/marketplace/contracts/provider.nim b/archivist/marketplace/contracts/provider.nim index 0c54c79e5..9cf09f2ce 100644 --- a/archivist/marketplace/contracts/provider.nim +++ b/archivist/marketplace/contracts/provider.nim @@ -120,4 +120,11 @@ proc pastBlockTag*( provider: Provider, blocksAgo: int ): Future[BlockTag] {.async: (raises: [ProviderError, CancelledError]).} = let head = await provider.getBlockNumber() - return BlockTag.init(head - blocksAgo.abs.u256) + let blocksAgoU = blocksAgo.abs.u256 + # Clamp to genesis: a fresh chain may be shorter than the requested window. + let fromBlock = + if blocksAgoU >= head: + 0.u256 + else: + head - blocksAgoU + return BlockTag.init(fromBlock) diff --git a/archivist/marketplace/purchasing.nim b/archivist/marketplace/purchasing.nim index 5f79862ff..173d4b39a 100644 --- a/archivist/marketplace/purchasing.nim +++ b/archivist/marketplace/purchasing.nim @@ -36,10 +36,19 @@ proc new*( ) proc load*(purchasing: Purchasing) {.async.} = - let marketplace = purchasing.marketplace - let requestIds = await marketplace.myRequests() - for requestId in requestIds: - let purchase = Purchase.new(requestId, purchasing.marketplace, purchasing.clock) + # Re-adopt requests from the onchain StorageRequested history after a restart. + let requestDurationLimit = purchasing.marketplace.requestDurationLimit() + let blocksAgo = int(requestDurationLimit.u64 div 12) + let requestEvents = + await purchasing.marketplace.queryPastStorageRequestedEvents(blocksAgo = blocksAgo) + let signer = await purchasing.marketplace.getSigner() + for event in requestEvents: + without request =? (await purchasing.marketplace.getRequest(event.requestId)): + continue + if request.client != signer: + continue + let purchase = + Purchase.new(event.requestId, purchasing.marketplace, purchasing.clock) purchase.load() purchasing.purchases[purchase.id] = purchase diff --git a/archivist/marketplace/purchasing/states/errored.nim b/archivist/marketplace/purchasing/states/errored.nim index 53c37221f..04fd56dca 100644 --- a/archivist/marketplace/purchasing/states/errored.nim +++ b/archivist/marketplace/purchasing/states/errored.nim @@ -24,7 +24,10 @@ method run*( try: await purchase.errorBackoff.applyDelay() - if purchase.requestId in await purchase.marketplace.myRequests(): + if request =? (await purchase.marketplace.getRequest(purchase.requestId)) and + request.client == (await purchase.marketplace.getSigner()) and + requestState =? (await purchase.marketplace.requestState(purchase.requestId)) and + requestState notin {RequestState.Failed, RequestState.Finished}: debug "Errored request is in myRequests. Restarting state machine" return some State(PurchaseUnknown()) else: diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index 067f5f998..38eb23d6c 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -143,8 +143,13 @@ proc getSlot*(sales: Sales, slotId: SlotId): Future[?SalesSlot] {.async.} = ) proc load(sales: Sales) {.async.} = - for slotId in await sales.context.marketplace.mySlots(): - let slotInfo = SlotInfo.init(slotId) + # Re-adopt slots from the onchain SlotFilled history after a restart. + let requestDurationLimit = sales.context.marketplace.requestDurationLimit() + let blocksAgo = int(requestDurationLimit.u64 div 12) + let slotFilledEvents = + await sales.context.marketplace.queryPastSlotFilledEvents(blocksAgo = blocksAgo) + for event in slotFilledEvents: + let slotInfo = SlotInfo.init(slotId(event.requestId, event.slotIndex)) let agent = newSalesAgent(sales.context, slotInfo) agent.onCleanUp = proc(reprocessSlot = false) {.async: (raises: []).} = diff --git a/archivist/marketplace/sales/states/errored.nim b/archivist/marketplace/sales/states/errored.nim index 77ece5b54..315a1dd3c 100644 --- a/archivist/marketplace/sales/states/errored.nim +++ b/archivist/marketplace/sales/states/errored.nim @@ -45,7 +45,8 @@ method run*( try: await data.errorBackoff.applyDelay() - if data.slotInfo.slotId in await marketplace.mySlots(): + let host = await marketplace.getHost(data.slotInfo.slotId) + if host.isSome and host.unsafeGet == (await marketplace.getSigner()): debug "Errored slot is in MySlots. Restarting state machine..." return some State(SaleUnknown()) diff --git a/archivist/marketplace/sales/states/slotreserving.nim b/archivist/marketplace/sales/states/slotreserving.nim index 615dfc0a2..a6c9589d4 100644 --- a/archivist/marketplace/sales/states/slotreserving.nim +++ b/archivist/marketplace/sales/states/slotreserving.nim @@ -43,7 +43,8 @@ method run*( try: let canReserve = await marketplace.canReserveSlot(slot.request.id, slot.slotIndex) - if canReserve: + let collateral = slot.request.ask.collateralPerSlot() + if canReserve and (await marketplace.tokensAvailable()) >= collateral.u256(): try: trace "Reserving slot" await marketplace.reserveSlot(slot.request.id, slot.slotIndex) diff --git a/tests/archivist/helpers/mockmarketplace.nim b/tests/archivist/helpers/mockmarketplace.nim index 707e530fb..bfb9d2c6b 100644 --- a/tests/archivist/helpers/mockmarketplace.nim +++ b/tests/archivist/helpers/mockmarketplace.nim @@ -48,6 +48,7 @@ type subscriptions: Subscriptions config*: MarketplaceConfig canReserveSlot*: bool + tokens*: UInt256 errorOnReserveSlot*: ?(ref MarketplaceError) errorOnFillSlot*: ?(ref MarketplaceError) errorOnFreeSlot*: ?(ref MarketplaceError) @@ -130,7 +131,11 @@ proc new*(_: type MockMarketplace, clock: Clock = MockClock.new()): MockMarketpl requestDurationLimit: StorageDuration.init(60 * 60 * 24 * 30.stuint(40)), ) MockMarketplace( - signer: Address.example, config: config, canReserveSlot: true, clock: clock + signer: Address.example, + config: config, + canReserveSlot: true, + tokens: high(UInt256), + clock: clock, ) method getSigner*( @@ -393,6 +398,11 @@ method canReserveSlot*( ): Future[bool] {.async.} = return marketplace.canReserveSlot +method tokensAvailable*( + marketplace: MockMarketplace +): Future[UInt256] {.async: (raises: [CancelledError, MarketplaceError]).} = + return marketplace.tokens + func setCanReserveSlot*(marketplace: MockMarketplace, canReserveSlot: bool) = marketplace.canReserveSlot = canReserveSlot @@ -520,7 +530,8 @@ method queryPastStorageRequestedEvents*( StorageRequested( requestId: request.id, ask: request.ask, - expiry: marketplace.requestExpiry[request.id], + expiry: + marketplace.requestExpiry.getOrDefault(request.id, StorageTimestamp.init(0)), ) ) @@ -532,7 +543,8 @@ method queryPastStorageRequestedEvents*( StorageRequested( requestId: request.id, ask: request.ask, - expiry: marketplace.requestExpiry[request.id], + expiry: + marketplace.requestExpiry.getOrDefault(request.id, StorageTimestamp.init(0)), ) ) diff --git a/tests/archivist/marketplace/sales/states/testerrored.nim b/tests/archivist/marketplace/sales/states/testerrored.nim index 61bc1b20a..0a1c9c213 100644 --- a/tests/archivist/marketplace/sales/states/testerrored.nim +++ b/tests/archivist/marketplace/sales/states/testerrored.nim @@ -53,7 +53,9 @@ asyncchecksuite "sales state 'errored'": test "transits to unknown state when slot is active": let me = await marketplace.getSigner() - marketplace.activeSlots[me] = @[slot.id] + marketplace.filled.add( + MockSlot(requestId: request.id, slotIndex: slotIndex, host: me) + ) let nextState = await runState() check !nextState of SaleUnknown @@ -74,5 +76,6 @@ asyncchecksuite "sales state 'errored'": check storage.deleteSlotCalls == @[(request.content.cid, slotIndex)] test "transits to error state when slots call fails": + marketplace.errorOnGetHost = some newException(MarketplaceError, "boom") let nextState = await runState() check !nextState of SaleErrored diff --git a/tests/archivist/marketplace/testpurchasing.nim b/tests/archivist/marketplace/testpurchasing.nim index 3c65191ec..55931278d 100644 --- a/tests/archivist/marketplace/testpurchasing.nim +++ b/tests/archivist/marketplace/testpurchasing.nim @@ -133,9 +133,10 @@ suite "Purchasing state machine": test "loads active purchases from marketplace": let me = await marketplace.getSigner() - let request1, request2, request3 = StorageRequest.example + var request1, request2, request3 = StorageRequest.example + request1.client = me + request2.client = me marketplace.requested = @[request1, request2, request3] - marketplace.activeRequests[me] = @[request1.id, request2.id] await purchasing.load() check isSome purchasing.getPurchase(PurchaseId(request1.id)) check isSome purchasing.getPurchase(PurchaseId(request2.id)) @@ -143,10 +144,13 @@ suite "Purchasing state machine": test "loads correct purchase.future state for purchases from marketplace": let me = await marketplace.getSigner() - let request1, request2, request3, request4, request5 = StorageRequest.example + var request1, request2, request3, request4, request5 = StorageRequest.example + request1.client = me + request2.client = me + request3.client = me + request4.client = me + request5.client = me marketplace.requested = @[request1, request2, request3, request4, request5] - marketplace.activeRequests[me] = - @[request1.id, request2.id, request3.id, request4.id, request5.id] marketplace.requestState[request1.id] = RequestState.New marketplace.requestState[request2.id] = RequestState.Started marketplace.requestState[request3.id] = RequestState.Cancelled @@ -261,11 +265,12 @@ suite "Purchasing state machine": check next.isNone test "restarts state machine when error occurs while request is active": - let request = StorageRequest.example + var request = StorageRequest.example + request.client = await marketplace.getSigner() let purchase = Purchase.new(request, marketplace, clock) let error = newException(CatchableError, "some error") - let me = await marketplace.getSigner() - marketplace.activeRequests[me] = @[request.id] + marketplace.requested.add(request) + marketplace.requestState[request.id] = RequestState.New let next = await PurchaseErrored(error: error).run(purchase) check !next of PurchaseUnknown diff --git a/vendor/nimble/contractabi b/vendor/nimble/contractabi index e9092906a..1c41c22e6 160000 --- a/vendor/nimble/contractabi +++ b/vendor/nimble/contractabi @@ -1 +1 @@ -Subproject commit e9092906a5b10de1f83e48c6e8b92ac5acfa3a6e +Subproject commit 1c41c22e6409d43b08ff37821fc5d4cff62a9d0c From 90a23b03240499ea594fa6a2b8b0898f772e499b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sun, 9 Aug 2026 10:51:44 -0600 Subject: [PATCH 06/14] Filter slot re-adoption to slots we actually filled The event-based sales reload created sale agents for every SlotFilled event in the window, including other hosts' fills, so the sales slots list could return foreign slots that 404 on lookup and break clients asking for them. Look up each slot's onchain host and skip slots that are not ours. --- archivist/marketplace/sales.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index 38eb23d6c..ef822163a 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -148,8 +148,15 @@ proc load(sales: Sales) {.async.} = let blocksAgo = int(requestDurationLimit.u64 div 12) let slotFilledEvents = await sales.context.marketplace.queryPastSlotFilledEvents(blocksAgo = blocksAgo) + let signer = await sales.context.marketplace.getSigner() for event in slotFilledEvents: - let slotInfo = SlotInfo.init(slotId(event.requestId, event.slotIndex)) + let slotId = slotId(event.requestId, event.slotIndex) + # Only re-adopt slots we actually filled; other hosts' fills are not our sale. + without host =? (await sales.context.marketplace.getHost(slotId)): + continue + if host != signer: + continue + let slotInfo = SlotInfo.init(slotId) let agent = newSalesAgent(sales.context, slotInfo) agent.onCleanUp = proc(reprocessSlot = false) {.async: (raises: []).} = From 39510a5f6892f63091cb0ad33a6e4ef9be0fc19b Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sun, 9 Aug 2026 11:44:16 -0600 Subject: [PATCH 07/14] Process at most one slot per request The onchain reservation allows several hosts per slot with first-fill-wins, so a host with three queue workers could reserve most slots of a request and then race other hosts on the same slot. Skip queue items for a request that already has an agent. --- archivist/marketplace/sales.nim | 10 ++++++++++ tests/archivist/marketplace/sales/testsales.nim | 14 +++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index ef822163a..6416124fd 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -105,6 +105,16 @@ proc processSlot( sales: Sales, item: SlotQueueItem ) {.async: (raises: [CancelledError]).} = debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex + + # The onchain reservation allows several hosts per slot with first-fill-wins, + # so without a cap a single host can reserve most slots of a request and then + # race other hosts on the same slot. Process at most one slot per request. + for agent in sales.agents: + if agent.data.slotInfo.requestId == some item.requestId: + debug "Already processing a slot of this request, skipping", + requestId = item.requestId, slot = item.slotIndex + return + var slotInfo = SlotInfo.init(item.requestId, item.slotIndex) slotInfo.ask = item.ask let agent = newSalesAgent(sales.context, slotInfo, some item) diff --git a/tests/archivist/marketplace/sales/testsales.nim b/tests/archivist/marketplace/sales/testsales.nim index 93f426651..61db2b17a 100644 --- a/tests/archivist/marketplace/sales/testsales.nim +++ b/tests/archivist/marketplace/sales/testsales.nim @@ -257,15 +257,11 @@ asyncchecksuite "Sales": check eventually queue.len > 0 # queue starts paused, allow items to be added to the queue check eventually queue.paused - # The first processed item will be will have been re-pushed. Then, once this - # item is processed by the queue, the queue will be paused. This test could - # check item existence in the queue, but that would require inspecting - # onProcessSlot to see which item was first, and overridding onProcessSlot - # will prevent the queue working as expected in the Sales module. - check eventually queue.len == 4 - - for item in items: - check queue.contains(item) + # Only one slot per request is processed: the remaining items are skipped + # while an agent for the request exists. One item stays parked in the + # queue (the paused worker readds the item it picked up). + check eventually queue.paused + check queue.len == 1 test "retrieves and stores data locally": await setAvailability() From fbd5d26f617efd09fa4e9cad8ba03249710f31d3 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sun, 9 Aug 2026 14:47:14 -0600 Subject: [PATCH 08/14] Revert "Process at most one slot per request" This reverts commit ba1d2d4190ef76f1fc50d8ed7b5db48fa8a23a7f. --- archivist/marketplace/sales.nim | 10 ---------- tests/archivist/marketplace/sales/testsales.nim | 14 +++++++++----- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index 6416124fd..ef822163a 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -105,16 +105,6 @@ proc processSlot( sales: Sales, item: SlotQueueItem ) {.async: (raises: [CancelledError]).} = debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex - - # The onchain reservation allows several hosts per slot with first-fill-wins, - # so without a cap a single host can reserve most slots of a request and then - # race other hosts on the same slot. Process at most one slot per request. - for agent in sales.agents: - if agent.data.slotInfo.requestId == some item.requestId: - debug "Already processing a slot of this request, skipping", - requestId = item.requestId, slot = item.slotIndex - return - var slotInfo = SlotInfo.init(item.requestId, item.slotIndex) slotInfo.ask = item.ask let agent = newSalesAgent(sales.context, slotInfo, some item) diff --git a/tests/archivist/marketplace/sales/testsales.nim b/tests/archivist/marketplace/sales/testsales.nim index 61db2b17a..93f426651 100644 --- a/tests/archivist/marketplace/sales/testsales.nim +++ b/tests/archivist/marketplace/sales/testsales.nim @@ -257,11 +257,15 @@ asyncchecksuite "Sales": check eventually queue.len > 0 # queue starts paused, allow items to be added to the queue check eventually queue.paused - # Only one slot per request is processed: the remaining items are skipped - # while an agent for the request exists. One item stays parked in the - # queue (the paused worker readds the item it picked up). - check eventually queue.paused - check queue.len == 1 + # The first processed item will be will have been re-pushed. Then, once this + # item is processed by the queue, the queue will be paused. This test could + # check item existence in the queue, but that would require inspecting + # onProcessSlot to see which item was first, and overridding onProcessSlot + # will prevent the queue working as expected in the Sales module. + check eventually queue.len == 4 + + for item in items: + check queue.contains(item) test "retrieves and stores data locally": await setAvailability() From 34b9b7abb74675203ac1be57d237c31fe68062ab Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 10 Aug 2026 16:53:25 -0600 Subject: [PATCH 09/14] Reduce block and batch sizes to bound memory amplification 128 KiB blocks (from 512 KiB) plus smaller store/serve batches cut the in-flight peaks that glibc's arena retention pins as RSS: upload 2 GiB -> 512 MiB, serve 72 MiB/peer -> 6 MiB/peer. Old datasets still decode via manifest blockSize. --- archivist/archivisttypes.nim | 2 +- archivist/blockexchange/engine/engine.nim | 4 ++-- archivist/erasure/erasure.nim | 9 ++------- archivist/node.nim | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/archivist/archivisttypes.nim b/archivist/archivisttypes.nim index 72baaa392..c202499ce 100644 --- a/archivist/archivisttypes.nim +++ b/archivist/archivisttypes.nim @@ -26,7 +26,7 @@ export tables const # Size of blocks for storage / network exchange, - DefaultBlockSize* = 512.KiBs + DefaultBlockSize* = 128.KiBs DefaultCellSize* = 2.KiBs # Proving defaults diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index e9e38c5c0..535599df6 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -54,8 +54,8 @@ const DefaultConcurrentTasks = 30 DefaultWantBlockBatchSize = DefaultMaxBatchBlocks # Serve slice sized so a full delivery message stays under the receiver's - # 32MB TCP window: 48 x 512KiB blocks = 24MiB. - DefaultMaxServeBatchBlocks = 48 + # 32MB TCP window: 16 x 128KiB blocks = 2MiB. + DefaultMaxServeBatchBlocks = 16 DefaultWantBlockBatchTimeout = 5.millis DiscoveryRateLimit = 3.seconds PresenceBatchSize = DefaultMaxWantListBatchSize diff --git a/archivist/erasure/erasure.nim b/archivist/erasure/erasure.nim index 0bc137046..19fec02b3 100644 --- a/archivist/erasure/erasure.nim +++ b/archivist/erasure/erasure.nim @@ -528,10 +528,7 @@ proc leopardDecodeTask*( ctx[].result = ThreadSpawnRes[seq[seq[byte]]].ok(move recovered) proc decodeInternal( - self: Erasure, - encodedTreeCid: Cid, - targetCid: Cid, - params: EncodingParams, + self: Erasure, encodedTreeCid: Cid, targetCid: Cid, params: EncodingParams ): Future[?!(ref seq[Cid], seq[Natural])] {.async: (raises: [CancelledError]).} = logScope: encodedTreeCid = encodedTreeCid @@ -699,9 +696,7 @@ proc repair*( body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = let (cids, _) = - ?await self.decodeInternal( - encoded.treeCid, encoded.originalTreeCid, params - ) + ?await self.decodeInternal(encoded.treeCid, encoded.originalTreeCid, params) tree = # sync build: root comparison only (decode paths do not stream) ?ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]) diff --git a/archivist/node.nim b/archivist/node.nim index 75bdc9f3f..d37b09c30 100644 --- a/archivist/node.nim +++ b/archivist/node.nim @@ -59,7 +59,7 @@ logScope: const DefaultFetchBatch = 128 - DefaultStoreBatch* = 1024 ## Number of blocks to batch when storing data + DefaultStoreBatch* = 256 ## Number of blocks to batch when storing data MaxInFlightBatches = 4 ## Maximum concurrent batch flushes for bounded parallelism type From b4a7b3d2d60d67f562d4b074f14dfa66c13cee66 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 10 Aug 2026 17:18:16 -0600 Subject: [PATCH 10/14] Double serve batch to 32 blocks to recover download throughput --- archivist/blockexchange/engine/engine.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index 535599df6..f68b3d01b 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -54,8 +54,8 @@ const DefaultConcurrentTasks = 30 DefaultWantBlockBatchSize = DefaultMaxBatchBlocks # Serve slice sized so a full delivery message stays under the receiver's - # 32MB TCP window: 16 x 128KiB blocks = 2MiB. - DefaultMaxServeBatchBlocks = 16 + # 32MB TCP window: 32 x 128KiB blocks = 4MiB. + DefaultMaxServeBatchBlocks = 32 DefaultWantBlockBatchTimeout = 5.millis DiscoveryRateLimit = 3.seconds PresenceBatchSize = DefaultMaxWantListBatchSize From 2212b920fac2d371db1c8e28d50878e4847fe602 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 10 Aug 2026 18:01:53 -0600 Subject: [PATCH 11/14] Retry missing blocks in store stream instead of aborting A single transient fetch miss (peer dropped a want-batch) or a store CAS retry-out aborted the whole download stream. Re-fetch missing indices up to 5 times with backoff before failing. --- archivist/streams/storestream.nim | 36 +++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/archivist/streams/storestream.nim b/archivist/streams/storestream.nim index 61c8d5a5a..d7b266ef9 100644 --- a/archivist/streams/storestream.nim +++ b/archivist/streams/storestream.nim @@ -30,6 +30,12 @@ logScope: const StoreStreamTrackerName* = "StoreStream" +# A block batch may miss blocks under load (a peer drops a want-batch, or a +# concurrent store write retries out). Retry the fetch before aborting the +# stream - one transient miss should not kill a whole download. +const DefaultStreamRetries* = 5 +const DefaultStreamRetryDelay = 50.millis + type # Make SeekableStream from a sequence of blocks stored in Manifest # (only original file data - see StoreStream.size) @@ -100,11 +106,33 @@ method readOnce*( trace "No blocks returned from store!" raise newLPStreamReadError(newException(IOError, "No blocks returned from store!")) - # Build a lookup table from block CID to block data for ordered copying - # We copy block by block in index order using single-block getBlock fallback - # if a block is missing from the batch result. + # Build a lookup table from block index to block data, retrying any indices + # that came back missing (transient fetch/store misses under load). + var blocksMap = blocks.toTable + var missing: seq[Natural] + for idx in indices: + if not blocksMap.hasKey(idx): + missing.add(idx) + + for attempt in 1 .. DefaultStreamRetries: + if missing.len == 0: + break + trace "Retrying missing blocks", missing = missing.len, attempt + without retried =? (await self.store.getBlocks(treeCid, missing)).tryGet.catch, err: + trace "Unable to retry blocks from store", err = err.msg + break + for (idx, blk) in retried: + blocksMap[idx] = blk + var stillMissing: seq[Natural] + for idx in missing: + if not blocksMap.hasKey(idx): + stillMissing.add(idx) + missing = stillMissing + if missing.len > 0 and attempt < DefaultStreamRetries: + await sleepAsync(DefaultStreamRetryDelay) + + # Copy block by block in index order var read = 0 - let blocksMap = blocks.toTable for idx in indices: if self.atEof or read >= nbytes: From 11134e81d85ea93afd19abbbbe273ec76e8c4933 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 10 Aug 2026 18:25:06 -0600 Subject: [PATCH 12/14] Raise CAS retries on contended leaf-block and counter flows Concurrent putBlocks on the same treeCid (swarm downloads from many peers) CAS-conflict on the shared overlay/refCount records; the 10-retry bound could exhaust under the burst. 30 retries let the writer storm drain; the middleware refetches and re-applies on fresh values. --- archivist/stores/repostore/operations.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivist/stores/repostore/operations.nim b/archivist/stores/repostore/operations.nim index 02377111b..6c66729cb 100644 --- a/archivist/stores/repostore/operations.nim +++ b/archivist/stores/repostore/operations.nim @@ -146,7 +146,7 @@ proc updateCounters*( success toSeq(refreshed.values) trace "Updating counters", quotaDelta, reservedDelta, blocksDelta - ?await self.metaDs.tryPutAtomic(updates, maxRetries = 10, updateCountersMiddleware) + ?await self.metaDs.tryPutAtomic(updates, maxRetries = 30, updateCountersMiddleware) if quotaDelta != 0: self.quotaUsage.used = max(0, self.quotaUsage.used.int + quotaDelta).NBytes @@ -482,7 +482,7 @@ proc putLeafBlockMetaImpl( trace "Put or update leaf and block metadata", treeCid, recordsCount = updates.len if err =? ( - await self.metaDs.tryPutAtomic(updates, maxRetries = 10, putLeafAndBlockMetaAtomic) + await self.metaDs.tryPutAtomic(updates, maxRetries = 30, putLeafAndBlockMetaAtomic) ).errorOption: trace "Unable to put or update leaf and block metadata", error = err.msg return failure(err) From ea10326dee3dc7594076a2e48eb24b05e6369a9d Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 12 Aug 2026 16:09:15 -0600 Subject: [PATCH 13/14] Pin glibc mmap threshold below block size to release hoarded memory glibc's mmap-threshold ratchet demotes the 128 KiB block buffers to the heap, where interleaved small allocations pin the arena tops and freed chunks never return to the OS - RSS stays at the historical peak forever (measured: 627 MB RSS -> 61 MB after malloc_trim(0)). Pinning the threshold to 64 KiB keeps block buffers on the mmap path so frees return memory at once, and disables the ratchet. Makes the 20-node x 300 MB StreamOneToMany swarm leg pass with flat memory (nodes ~700 MB vs 2.5 GB hoarded); threshold beats per-poll malloc_trim in the A/B (no measurable speed cost, trim was 7-10% slower at swarm scale). --- archivist.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/archivist.nim b/archivist.nim index 138517204..bdbc443ce 100644 --- a/archivist.nim +++ b/archivist.nim @@ -39,6 +39,14 @@ when isMainModule: when defined(posix): import system/ansi_c + when defined(linux): + # Pin the mmap threshold below the block size: block buffers stay on + # the mmap path and return memory to the OS on free. Setting it + # explicitly also disables the dynamic ratchet that demotes them. + const M_MMAP_THRESHOLD = -3 + proc mallopt(param: cint, value: cint): cint {.importc, header: "".} + discard mallopt(M_MMAP_THRESHOLD, 64 * 1024) + type NodeStatus {.pure.} = enum Stopped Stopping From 308ee33795fc3ff237d9130bf0f59054a64e7211 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 19 Aug 2026 09:17:40 -0600 Subject: [PATCH 14/14] Re-pin contractabi to reachable main pin (e9092906) The branch carried an old contractabi gitlink (1c41c22e) that was never pushed to the durability-labs fork - CI submodule fetch fails with 'not our ref'. Main's pin (e9092906, v0.7.4) is reachable and the branch was rebased on it; align the pin and the CI submodule stage passes. --- vendor/nimble/contractabi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nimble/contractabi b/vendor/nimble/contractabi index 1c41c22e6..e9092906a 160000 --- a/vendor/nimble/contractabi +++ b/vendor/nimble/contractabi @@ -1 +1 @@ -Subproject commit 1c41c22e6409d43b08ff37821fc5d4cff62a9d0c +Subproject commit e9092906a5b10de1f83e48c6e8b92ac5acfa3a6e