diff --git a/archivist.nim b/archivist.nim index 138517204..bdbc443ce 100644 --- a/archivist.nim +++ b/archivist.nim @@ -39,6 +39,14 @@ when isMainModule: when defined(posix): import system/ansi_c + when defined(linux): + # Pin the mmap threshold below the block size: block buffers stay on + # the mmap path and return memory to the OS on free. Setting it + # explicitly also disables the dynamic ratchet that demotes them. + const M_MMAP_THRESHOLD = -3 + proc mallopt(param: cint, value: cint): cint {.importc, header: "".} + discard mallopt(M_MMAP_THRESHOLD, 64 * 1024) + type NodeStatus {.pure.} = enum Stopped Stopping diff --git a/archivist/archivist.nim b/archivist/archivist.nim index 0b38c2738..765ea5918 100644 --- a/archivist/archivist.nim +++ b/archivist/archivist.nim @@ -57,7 +57,7 @@ type NodePrivateKey* = libp2p.PrivateKey # alias -proc connectMarketplace(s: NodeServer) {.async.} = +proc connectMarketplace(s: NodeServer) {.async: (raises: [CancelledError]).} = let config = s.config if config.persistence: @@ -65,28 +65,40 @@ proc connectMarketplace(s: NodeServer) {.async.} = error "Persistence enabled, but no Ethereum private key was set" quit QuitFailure - let marketplaceResult = await MarketplaceNode.connect( - ethProviderUrl = config.ethProvider, - ethPrivateKeyFile = ethPrivateKeyFile, - datastore = s.repoStore.metaDs, - storage = MarketplaceStorage.new(s.archivistNode, s.repoStore), - options = MarketplaceOptions( - marketplaceAddress: config.marketplaceAddress, - maxPriorityFeePerGas: config.maxPriorityFeePerGas, - requestCacheSize: config.marketplaceRequestCacheSize, - validationEnabled: config.validator, - validationMaxSlots: some config.validatorMaxSlots, - validationGroups: config.validatorGroups, - validationGroupIndex: some config.validatorGroupIndex, - useSystemClock: config.useSystemClock, - ), - ) + # The RPC node may not be reachable on the first attempt (e.g. the geth + # endpoint is still propagating after a restart); retry before giving up. + var marketplace: ?MarketplaceNode + for attempt in 1 .. config.marketplaceConnectRetries: + without res =? ( + await MarketplaceNode.connect( + ethProviderUrl = config.ethProvider, + ethPrivateKeyFile = ethPrivateKeyFile, + datastore = s.repoStore.metaDs, + storage = MarketplaceStorage.new(s.archivistNode, s.repoStore), + options = MarketplaceOptions( + marketplaceAddress: config.marketplaceAddress, + maxPriorityFeePerGas: config.maxPriorityFeePerGas, + requestCacheSize: config.marketplaceRequestCacheSize, + validationEnabled: config.validator, + validationMaxSlots: some config.validatorMaxSlots, + validationGroups: config.validatorGroups, + validationGroupIndex: some config.validatorGroupIndex, + useSystemClock: config.useSystemClock, + ), + ) + ), err: + error "Unable to connect to marketplace", error = err.msg, attempt = attempt + if attempt < config.marketplaceConnectRetries: + await sleepAsync(config.marketplaceConnectRetryDelay) + continue + + marketplace = res.some + break - without marketplace =? marketplaceResult, err: - error "Unable to connect to marketplace", error = err.msg + without marketplaceNode =? marketplace: quit QuitFailure - s.archivistNode.marketplace = marketplace + s.archivistNode.marketplace = marketplaceNode proc start*(s: NodeServer) {.async.} = trace "Starting node", config = $s.config diff --git a/archivist/archivisttypes.nim b/archivist/archivisttypes.nim index 72baaa392..c202499ce 100644 --- a/archivist/archivisttypes.nim +++ b/archivist/archivisttypes.nim @@ -26,7 +26,7 @@ export tables const # Size of blocks for storage / network exchange, - DefaultBlockSize* = 512.KiBs + DefaultBlockSize* = 128.KiBs DefaultCellSize* = 2.KiBs # Proving defaults diff --git a/archivist/blockexchange/engine/advertiser.nim b/archivist/blockexchange/engine/advertiser.nim index a7db1bb20..8c60e2287 100644 --- a/archivist/blockexchange/engine/advertiser.nim +++ b/archivist/blockexchange/engine/advertiser.nim @@ -35,9 +35,8 @@ logScope: const DefaultConcurrentAdvertRequests = 10 DefaultAdvertiseLoopSleep = 30.minutes - DefaultMinAdvertisePeers = 16 + DefaultMinAdvertisePeers* = 16 DefaultAdvertiseRetrySleep = 30.seconds - DefaultAdvertiseRetryWindow = 10.minutes type Advertiser* = ref object of RootObj localStore*: BlockStore # Local block store for this instance @@ -55,8 +54,6 @@ type Advertiser* = ref object of RootObj pendingAdvRetries: Table[Cid, Future[void]] # Delayed sparse retry tasks minAdvertisePeers: int # Desired routing-table size before advertising advertiseRetrySleep: Duration # Delay before retrying sparse startup advertise - advertiseRetryWindow: Duration # Startup window for sparse advertise retries - startedAt: Moment # Time advertiser started proc addCidToQueue(b: Advertiser, cid: Cid) {.async: (raises: [CancelledError]).} = if cid notin b.advertiseQueue: @@ -113,18 +110,14 @@ proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} = info "Exiting advertise task loop" -proc hasElapsed(since: Moment, dur: Duration): bool = - (Moment.now() - since) >= dur - proc shouldRetryAdvertise(b: Advertiser): bool = - b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers and - not hasElapsed(b.startedAt, b.advertiseRetryWindow) + b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers proc delayedAdvertiseRetry(b: Advertiser, cid: Cid) {.async: (raises: []).} = try: await sleepAsync(b.advertiseRetrySleep) - if b.advertiserRunning and not hasElapsed(b.startedAt, b.advertiseRetryWindow): + if b.advertiserRunning and b.shouldRetryAdvertise(): trace "Requeueing advertisement retry", cid, nodes = b.discovery.nodesDiscovered(), target = b.minAdvertisePeers await b.addCidToQueue(cid) @@ -192,7 +185,6 @@ proc start*(b: Advertiser) {.async: (raises: []).} = doAssert(b.localStore.onBlockStored.isNone()) b.localStore.onBlockStored = onBlock.some - b.startedAt = Moment.now() for i in 0 ..< b.concurrentAdvReqs: let fut = b.processQueueLoop() b.trackedFutures.track(fut) @@ -225,7 +217,6 @@ proc new*( advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep, minAdvertisePeers = DefaultMinAdvertisePeers, advertiseRetrySleep = DefaultAdvertiseRetrySleep, - advertiseRetryWindow = DefaultAdvertiseRetryWindow, ): Advertiser = ## Create a advertiser instance ## @@ -240,5 +231,4 @@ proc new*( advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep, minAdvertisePeers: minAdvertisePeers, advertiseRetrySleep: advertiseRetrySleep, - advertiseRetryWindow: advertiseRetryWindow, ) diff --git a/archivist/blockexchange/engine/discovery.nim b/archivist/blockexchange/engine/discovery.nim index ad9f41956..99dcadab9 100644 --- a/archivist/blockexchange/engine/discovery.nim +++ b/archivist/blockexchange/engine/discovery.nim @@ -8,6 +8,7 @@ ## those terms. import std/sequtils +import std/sets import pkg/chronos import pkg/libp2p/cid @@ -37,6 +38,10 @@ const DefaultDiscoveryTimeout = 1.minutes DefaultMinPeersPerBlock = 3 DefaultDiscoveryLoopSleep = 3.seconds + DefaultDiscoverBackoffBase = 3.seconds + DefaultDiscoverBackoffMax = 60.seconds + DefaultDiscoverBackoffMaxMult = 5 + # base << 5 (96s) exceeds the 60s cap, so the interval saturates there type DiscoveryEngine* = ref object of RootObj localStore*: BlockStore # Local block store for this instance @@ -51,12 +56,24 @@ type DiscoveryEngine* = ref object of RootObj trackedFutures*: TrackedFutures # Tracked Discovery tasks futures minPeersPerBlock*: int # Max number of peers with block discoveryLoopSleep: Duration # Discovery loop sleep + discoverBackoffBase: Duration # Backoff base for failed discovery requests + discoverBackoffMax: Duration # Backoff cap for failed discovery requests inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]] # Inflight discovery requests + discoverBackoff: Table[Cid, tuple[due: Moment, mult: int]] + # Earliest retry time and backoff multiplier per CID proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} = try: while b.discEngineRunning: + # Drop backoff state for blocks that are no longer wanted - the table + # must not grow with every cid ever queued. wantListCids covers leaf + # wants' tree cids, which also get discovery finds (searchForNewPeers). + let wanted = toHashSet(toSeq(b.pendingBlocks.wantListCids)) + for cid in toSeq(b.discoverBackoff.keys): + if cid notin wanted: + b.discoverBackoff.del(cid) + for cid in toSeq(b.pendingBlocks.wantListBlockCids): await b.discoveryQueue.put(cid) @@ -64,6 +81,17 @@ proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} = except CancelledError: trace "Discovery loop cancelled" +proc nextDiscoverBackoff(b: DiscoveryEngine, cid: Cid) {.raises: [].} = + ## Advance the per-CID discovery backoff after a failed or empty find: + ## increment the stored multiplier (capped at + ## DefaultDiscoverBackoffMaxMult), then the next attempt is due in + ## base * 2^mult, capped at discoverBackoffMax. + var mult = 0 + b.discoverBackoff.withValue(cid, entry): + mult = min(entry[].mult + 1, DefaultDiscoverBackoffMaxMult) + let interval = min(b.discoverBackoffBase * (1 shl mult), b.discoverBackoffMax) + b.discoverBackoff[cid] = (due: Moment.now() + interval, mult: mult) + proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = ## Run discovery tasks ## @@ -77,8 +105,13 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = continue let haves = b.peers.peersHave(cid) - if haves.len < b.minPeersPerBlock: + var notDue = false + b.discoverBackoff.withValue(cid, entry): + notDue = Moment.now() < entry[].due + if notDue: + continue + let request = b.discovery.find(cid) b.inFlightDiscReqs[cid] = request archivist_inflight_discovery.set(b.inFlightDiscReqs.len.int64) @@ -89,11 +122,17 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = if (await request.withTimeout(DefaultDiscoveryTimeout)) and peers =? (await request).catch: - let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data))) - - for i, f in dialed: - if f.failed: - await b.discovery.removeProvider(peers[i].data.peerId) + if peers.len > 0: + b.discoverBackoff.del(cid) + + let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data))) + for i, f in dialed: + if f.failed: + await b.discovery.removeProvider(peers[i].data.peerId) + else: + b.nextDiscoverBackoff(cid) + else: + b.nextDiscoverBackoff(cid) except CancelledError: trace "Discovery task cancelled" return @@ -154,6 +193,8 @@ proc new*( concurrentDiscReqs = DefaultConcurrentDiscRequests, discoveryLoopSleep = DefaultDiscoveryLoopSleep, minPeersPerBlock = DefaultMinPeersPerBlock, + discoverBackoffBase = DefaultDiscoverBackoffBase, + discoverBackoffMax = DefaultDiscoverBackoffMax, ): DiscoveryEngine = ## Create a discovery engine instance for advertising services ## @@ -169,4 +210,7 @@ proc new*( inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](), discoveryLoopSleep: discoveryLoopSleep, minPeersPerBlock: minPeersPerBlock, + discoverBackoffBase: discoverBackoffBase, + discoverBackoffMax: discoverBackoffMax, + discoverBackoff: initTable[Cid, tuple[due: Moment, mult: int]](), ) diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index 945771e28..f68b3d01b 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -54,8 +54,8 @@ const DefaultConcurrentTasks = 30 DefaultWantBlockBatchSize = DefaultMaxBatchBlocks # Serve slice sized so a full delivery message stays under the receiver's - # 32MB TCP window: 48 x 512KiB blocks = 24MiB. - DefaultMaxServeBatchBlocks = 48 + # 32MB TCP window: 32 x 128KiB blocks = 4MiB. + DefaultMaxServeBatchBlocks = 32 DefaultWantBlockBatchTimeout = 5.millis DiscoveryRateLimit = 3.seconds PresenceBatchSize = DefaultMaxWantListBatchSize @@ -210,9 +210,11 @@ proc refreshBlockKnowledge( return decision.wakeAfter of SendKind.Full: await self.sendBatchedWantList(peer, decision.wants.toSeq, full = true) + peer.markQueried() return Duration.none of SendKind.Delta: await self.sendBatchedWantList(peer, decision.wants.toSeq, full = false) + peer.markQueried() return Duration.none proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) = @@ -223,6 +225,16 @@ proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) = else: current = wake.some +proc hasUnsentWants(self: BlockExcEngine, peer: BlockExcPeerCtx): bool = + ## True when some wanted block has not been asked of this peer yet. + ## The query quarantine only suppresses re-asks of the same want list - + ## newly queued wants must still reach the peer (mirrors decideSend's + ## delta computation without mutating the circuit state). + for address in self.pendingBlocks.wantList: + if address notin peer.blocks and address notin peer.alreadySent: + return true + false + proc searchForNewPeers(self: BlockExcEngine, cid: Cid) = if self.lastDiscRequest + DiscoveryRateLimit < Moment.now(): archivist_block_exchange_discovery_requests.inc() @@ -235,20 +247,24 @@ proc evictPeer(self: BlockExcEngine, peer: PeerId) {.gcsafe, async: (raises: []) self.peers.remove(peer) proc topPeersForCid*(self: BlockExcEngine, cid: Cid, topK: int): seq[BlockExcPeerCtx] = - let candidates = self.peers.peersHave(cid) - if candidates.len == 0: - return @[] - let ranked = rankPeersByScore(candidates, cid) - if ranked.len == 0: - return @[] - ranked[0 ..< min(topK, ranked.len)] + let ranked = rankPeersByScore(self.peers.peersHave(cid), cid) + var top: seq[BlockExcPeerCtx] + for peer in ranked: + if top.len >= topK: + break + if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer): + top.add(peer) + top proc topPeersByAggregate*(self: BlockExcEngine, topK: int): seq[BlockExcPeerCtx] = - let allPeers = toSeq(self.peers.peers.values) - let ranked = rankPeersByAggregate(allPeers) - if ranked.len == 0: - return @[] - ranked[0 ..< min(topK, ranked.len)] + let ranked = rankPeersByAggregate(toSeq(self.peers.peers.values)) + var top: seq[BlockExcPeerCtx] + for peer in ranked: + if top.len >= topK: + break + if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer): + top.add(peer) + top proc failBlockRequest( self: BlockExcEngine, @@ -882,11 +898,7 @@ proc new*( candidates = self.peers.peersHave(cid) topPeers = if candidates.len > 0: - let ranked = rankPeersByScore(candidates, cid) - if ranked.len == 0: - @[] - else: - ranked[0 ..< min(DefaultWantHaveTopK, ranked.len)] + self.topPeersForCid(cid, DefaultWantHaveTopK) else: self.topPeersByAggregate(DefaultWantHaveTopK) @@ -899,6 +911,16 @@ proc new*( minWakeHint(minWake, fut.read()) except CatchableError: discard + else: + # Every candidate was queried recently - wake when the first quarantine + # expires instead of re-running selection in a tight loop. + let pool = + if candidates.len > 0: + candidates + else: + toSeq(self.peers.peers.values) + for peer in pool: + minWakeHint(minWake, peer.queryQuarantineLeft()) let discoveredPeer = selectKnownPeer(address) if not discoveredPeer.isNil: diff --git a/archivist/blockexchange/engine/peerscoring.nim b/archivist/blockexchange/engine/peerscoring.nim index 46ea8aafa..c44aae33c 100644 --- a/archivist/blockexchange/engine/peerscoring.nim +++ b/archivist/blockexchange/engine/peerscoring.nim @@ -38,6 +38,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC let s = effectiveScore(peer, cid, now) ranked.add((s, peer)) + # Randomize equal-score groups (stable sort preserves shuffle order) + Rng.instance.shuffle(ranked) ranked.sort( proc(a, b: auto): int = cmp(b[0], a[0]) @@ -46,6 +48,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC proc rankPeersByAggregate*(peers: seq[BlockExcPeerCtx]): seq[BlockExcPeerCtx] = var ranked = peers.mapIt((aggregateScore(it), it)) + # Randomize equal-score groups (stable sort preserves shuffle order) + Rng.instance.shuffle(ranked) ranked.sort( proc(a, b: auto): int = cmp(b[0], a[0]) diff --git a/archivist/blockexchange/peers/peercontext.nim b/archivist/blockexchange/peers/peercontext.nim index e541a426e..e6d86305b 100644 --- a/archivist/blockexchange/peers/peercontext.nim +++ b/archivist/blockexchange/peers/peercontext.nim @@ -72,6 +72,7 @@ type scores*: Table[Cid, PeerScore] # scores per dataset (cid) disconnected: Future[void] # completed when peer is removed from store + lastQueriedAt: Moment # when this peer was last asked about wanted blocks proc openWindow*(self: BlockExcPeerCtx, now = Moment.now()) = self.state = WantListState.SendAll @@ -149,6 +150,17 @@ proc markBlockAsNotSent*(self: BlockExcPeerCtx, address: BlockAddress) = ## a cancelled send never left the peer, so no cooldown applies. self.blocksSent.del(address) +proc markQueried*(self: BlockExcPeerCtx, now = Moment.now()) = + self.lastQueriedAt = now + +proc queryQuarantineLeft*( + self: BlockExcPeerCtx, + quarantine: Duration = DefaultWantHaveResendTimeout, + now = Moment.now(), +): ?Duration = + let left = (self.lastQueriedAt + quarantine) - now + if left.nanoseconds > 0: left.some else: Duration.none + proc clearSentIfStale*( self: BlockExcPeerCtx, address: BlockAddress, cooldown: Duration ): bool = @@ -201,6 +213,11 @@ proc ensureScoreFor*(self: BlockExcPeerCtx, cid: Cid): PeerScore = peerScore proc aggregateScore*(peer: BlockExcPeerCtx): float = + if peer.scores.len == 0: + # No evidence yet - baseline like the per-cid score path does, so a + # freshly discovered peer is not ranked below every known peer. + return ColdStartScore + var total = 0.0 for _, score in peer.scores: score.computeScore(peer.blocksRequested.len) @@ -211,6 +228,7 @@ proc aggregateScore*(peer: BlockExcPeerCtx): float = proc effectiveScore*(peer: BlockExcPeerCtx, cid: Cid, now: Moment): float = without score =? peer.scoreFor(cid): return ColdStartScore + score.computeScore(peer.blocksRequested.len) applyDecay(score.score, score.lastUpdated, peer.blocksRequested.len, now) diff --git a/archivist/conf.nim b/archivist/conf.nim index 31b88aa33..40b6f600d 100644 --- a/archivist/conf.nim +++ b/archivist/conf.nim @@ -48,6 +48,10 @@ export DefaultQuotaBytes, DefaultOverlayTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas +const + DefaultMarketplaceConnectRetries* = 10 + DefaultMarketplaceConnectRetryDelay* = 10 + type ThreadCount* = range[0 .. 256] proc defaultDataDir*(): string = @@ -331,6 +335,20 @@ type name: "use-system-clock" .}: bool + marketplaceConnectRetries* {. + desc: "Number of marketplace connection retries before giving up", + defaultValue: DefaultMarketplaceConnectRetries, + defaultValueDesc: $DefaultMarketplaceConnectRetries, + name: "marketplace-connect-retries" + .}: int + + marketplaceConnectRetryDelay* {. + desc: "Delay in seconds between marketplace connection retries", + defaultValue: DefaultMarketplaceConnectRetryDelay.seconds, + defaultValueDesc: $DefaultMarketplaceConnectRetryDelay, + name: "marketplace-connect-retry-delay" + .}: Duration + validator* {. desc: "Enables validator, requires an Ethereum node", defaultValue: false, diff --git a/archivist/discovery.nim b/archivist/discovery.nim index 0c609890a..c3f70363a 100644 --- a/archivist/discovery.nim +++ b/archivist/discovery.nim @@ -123,7 +123,7 @@ method provide*(d: Discovery, cid: Cid) {.async: (raises: [CancelledError]), bas except CatchableError as exc: warn "Error providing block", cid, exc = exc.msg -proc nodesDiscovered*(d: Discovery): int = +method nodesDiscovered*(d: Discovery): int {.base, gcsafe.} = if d.protocol.isNil: return 0 diff --git a/archivist/erasure/erasure.nim b/archivist/erasure/erasure.nim index a1fe5808c..19fec02b3 100644 --- a/archivist/erasure/erasure.nim +++ b/archivist/erasure/erasure.nim @@ -421,6 +421,7 @@ proc encode*( blocks: Natural, parity: Natural, strategy = SteppedStrategy, + expiry = ZeroSeconds, ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Encode a manifest into one that is erasure protected. ## @@ -474,6 +475,7 @@ proc encode*( ?await self.repoStore.withOverlay( manifest.treeCid, status = Repairing.some, + expiry = expiry, body = proc(): Future[?!Cid] {. closure, gcsafe, async: (raises: [CancelledError]) .} = @@ -658,7 +660,7 @@ proc decode*( return decoded.success proc repair*( - self: Erasure, encoded: Manifest + self: Erasure, encoded: Manifest, expiry = ZeroSeconds ): Future[?!void] {.async: (raises: [CancelledError]).} = ## Repair a protected manifest by reconstructing the full dataset ## @@ -690,6 +692,7 @@ proc repair*( ?await self.repoStore.withOverlay( encoded.originalTreeCid, status = Repairing.some, + expiry = expiry, body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = let (cids, _) = @@ -714,7 +717,11 @@ proc repair*( # blocks for a valid slot - this is higly inneficient # we need to either fix leopard or use another implementation let repaired = - ?(await self.encode(encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy)) + ?( + await self.encode( + encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy, expiry + ) + ) trace "Successfully re-encoded original dataset" if repaired.treeCid != encoded.treeCid: diff --git a/archivist/marketplace/abstractmarketplace.nim b/archivist/marketplace/abstractmarketplace.nim index 0d8744072..801a1cb25 100644 --- a/archivist/marketplace/abstractmarketplace.nim +++ b/archivist/marketplace/abstractmarketplace.nim @@ -230,6 +230,11 @@ method canReserveSlot*( ): Future[bool] {.base, async.} = raiseAssert("not implemented") +method tokensAvailable*( + marketplace: AbstractMarketplace +): Future[UInt256] {.base, async: (raises: [CancelledError, MarketplaceError]).} = + raiseAssert("not implemented") + method subscribeFulfillment*( marketplace: AbstractMarketplace, callback: OnFulfillment ): Future[Subscription] {.base, async.} = diff --git a/archivist/marketplace/contracts/onchainmarketplace.nim b/archivist/marketplace/contracts/onchainmarketplace.nim index 61276ec5a..140af0a47 100644 --- a/archivist/marketplace/contracts/onchainmarketplace.nim +++ b/archivist/marketplace/contracts/onchainmarketplace.nim @@ -366,6 +366,15 @@ method canReserveSlot*( convertEthersError("Unable to determine if slot can be reserved"): return await marketplace.contract.canReserveSlot(requestId, slotIndex) +method tokensAvailable*( + marketplace: OnChainMarketplace +): Future[UInt256] {.async: (raises: [CancelledError, MarketplaceError]).} = + convertEthersError("Failed to get token balance"): + let tokenAddress = await marketplace.contract.token() + let token = Erc20Token.new(tokenAddress, marketplace.signer) + let owner = await marketplace.signer.getAddress() + return await token.balanceOf(owner) + method subscribeRequests*( marketplace: OnChainMarketplace, callback: OnRequest ): Future[MarketSubscription] {.async.} = diff --git a/archivist/marketplace/contracts/provider.nim b/archivist/marketplace/contracts/provider.nim index 0c54c79e5..9cf09f2ce 100644 --- a/archivist/marketplace/contracts/provider.nim +++ b/archivist/marketplace/contracts/provider.nim @@ -120,4 +120,11 @@ proc pastBlockTag*( provider: Provider, blocksAgo: int ): Future[BlockTag] {.async: (raises: [ProviderError, CancelledError]).} = let head = await provider.getBlockNumber() - return BlockTag.init(head - blocksAgo.abs.u256) + let blocksAgoU = blocksAgo.abs.u256 + # Clamp to genesis: a fresh chain may be shorter than the requested window. + let fromBlock = + if blocksAgoU >= head: + 0.u256 + else: + head - blocksAgoU + return BlockTag.init(fromBlock) diff --git a/archivist/marketplace/purchasing.nim b/archivist/marketplace/purchasing.nim index 5f79862ff..173d4b39a 100644 --- a/archivist/marketplace/purchasing.nim +++ b/archivist/marketplace/purchasing.nim @@ -36,10 +36,19 @@ proc new*( ) proc load*(purchasing: Purchasing) {.async.} = - let marketplace = purchasing.marketplace - let requestIds = await marketplace.myRequests() - for requestId in requestIds: - let purchase = Purchase.new(requestId, purchasing.marketplace, purchasing.clock) + # Re-adopt requests from the onchain StorageRequested history after a restart. + let requestDurationLimit = purchasing.marketplace.requestDurationLimit() + let blocksAgo = int(requestDurationLimit.u64 div 12) + let requestEvents = + await purchasing.marketplace.queryPastStorageRequestedEvents(blocksAgo = blocksAgo) + let signer = await purchasing.marketplace.getSigner() + for event in requestEvents: + without request =? (await purchasing.marketplace.getRequest(event.requestId)): + continue + if request.client != signer: + continue + let purchase = + Purchase.new(event.requestId, purchasing.marketplace, purchasing.clock) purchase.load() purchasing.purchases[purchase.id] = purchase diff --git a/archivist/marketplace/purchasing/states/errored.nim b/archivist/marketplace/purchasing/states/errored.nim index 53c37221f..04fd56dca 100644 --- a/archivist/marketplace/purchasing/states/errored.nim +++ b/archivist/marketplace/purchasing/states/errored.nim @@ -24,7 +24,10 @@ method run*( try: await purchase.errorBackoff.applyDelay() - if purchase.requestId in await purchase.marketplace.myRequests(): + if request =? (await purchase.marketplace.getRequest(purchase.requestId)) and + request.client == (await purchase.marketplace.getSigner()) and + requestState =? (await purchase.marketplace.requestState(purchase.requestId)) and + requestState notin {RequestState.Failed, RequestState.Finished}: debug "Errored request is in myRequests. Restarting state machine" return some State(PurchaseUnknown()) else: diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index 067f5f998..ef822163a 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -143,7 +143,19 @@ proc getSlot*(sales: Sales, slotId: SlotId): Future[?SalesSlot] {.async.} = ) proc load(sales: Sales) {.async.} = - for slotId in await sales.context.marketplace.mySlots(): + # Re-adopt slots from the onchain SlotFilled history after a restart. + let requestDurationLimit = sales.context.marketplace.requestDurationLimit() + let blocksAgo = int(requestDurationLimit.u64 div 12) + let slotFilledEvents = + await sales.context.marketplace.queryPastSlotFilledEvents(blocksAgo = blocksAgo) + let signer = await sales.context.marketplace.getSigner() + for event in slotFilledEvents: + let slotId = slotId(event.requestId, event.slotIndex) + # Only re-adopt slots we actually filled; other hosts' fills are not our sale. + without host =? (await sales.context.marketplace.getHost(slotId)): + continue + if host != signer: + continue let slotInfo = SlotInfo.init(slotId) let agent = newSalesAgent(sales.context, slotInfo) diff --git a/archivist/marketplace/sales/states/errored.nim b/archivist/marketplace/sales/states/errored.nim index 77ece5b54..315a1dd3c 100644 --- a/archivist/marketplace/sales/states/errored.nim +++ b/archivist/marketplace/sales/states/errored.nim @@ -45,7 +45,8 @@ method run*( try: await data.errorBackoff.applyDelay() - if data.slotInfo.slotId in await marketplace.mySlots(): + let host = await marketplace.getHost(data.slotInfo.slotId) + if host.isSome and host.unsafeGet == (await marketplace.getSigner()): debug "Errored slot is in MySlots. Restarting state machine..." return some State(SaleUnknown()) diff --git a/archivist/marketplace/sales/states/slotreserving.nim b/archivist/marketplace/sales/states/slotreserving.nim index 615dfc0a2..a6c9589d4 100644 --- a/archivist/marketplace/sales/states/slotreserving.nim +++ b/archivist/marketplace/sales/states/slotreserving.nim @@ -43,7 +43,8 @@ method run*( try: let canReserve = await marketplace.canReserveSlot(slot.request.id, slot.slotIndex) - if canReserve: + let collateral = slot.request.ask.collateralPerSlot() + if canReserve and (await marketplace.tokensAvailable()) >= collateral.u256(): try: trace "Reserving slot" await marketplace.reserveSlot(slot.request.id, slot.slotIndex) diff --git a/archivist/node.nim b/archivist/node.nim index 2c4c2de89..d37b09c30 100644 --- a/archivist/node.nim +++ b/archivist/node.nim @@ -59,7 +59,7 @@ logScope: const DefaultFetchBatch = 128 - DefaultStoreBatch* = 1024 ## Number of blocks to batch when storing data + DefaultStoreBatch* = 256 ## Number of blocks to batch when storing data MaxInFlightBatches = 4 ## Maximum concurrent batch flushes for bounded parallelism type @@ -916,7 +916,7 @@ proc storeSlot*( self.networkStore, self.repoStore, leoEncoderProvider, leoDecoderProvider, self.taskpool, ) - if err =? (await erasure.repair(manifest)).errorOption: + if err =? (await erasure.repair(manifest, expiry)).errorOption: error "Unable to erasure decode repairing manifest", cid = manifest.treeCid, exc = err.msg return failure(err) diff --git a/archivist/rest/api.nim b/archivist/rest/api.nim index 855bdc971..9d61c14fd 100644 --- a/archivist/rest/api.nim +++ b/archivist/rest/api.nim @@ -155,15 +155,28 @@ proc retrieveCid( bytes += buff.len await resp.send(addr buff[0], buff.len) + + if bytes != contentLength.int: + # The response would end short of the declared Content-Length. Abort + # without finishing so the client sees the truncation loudly instead of + # a clean short body. + error "Response body shorter than declared", + cid = cid, bytes, declared = contentLength.int + return + await resp.finish() archivist_api_downloads.inc() except CancelledError as exc: raise exc except LPStreamError as exc: warn "Error streaming blocks", exc = exc.msg - resp.status = Http500 if resp.isPending(): + resp.status = Http500 await resp.sendBody(exc.msg) + else: + # The response already started - abort so the client sees the truncation + # loudly instead of a clean short body. + error "Response aborted mid-stream", cid = cid, bytes, error = exc.msg finally: info "Sent bytes", cid = cid, bytes if not lpStream.isNil: diff --git a/archivist/stores/repostore/operations.nim b/archivist/stores/repostore/operations.nim index 02377111b..6c66729cb 100644 --- a/archivist/stores/repostore/operations.nim +++ b/archivist/stores/repostore/operations.nim @@ -146,7 +146,7 @@ proc updateCounters*( success toSeq(refreshed.values) trace "Updating counters", quotaDelta, reservedDelta, blocksDelta - ?await self.metaDs.tryPutAtomic(updates, maxRetries = 10, updateCountersMiddleware) + ?await self.metaDs.tryPutAtomic(updates, maxRetries = 30, updateCountersMiddleware) if quotaDelta != 0: self.quotaUsage.used = max(0, self.quotaUsage.used.int + quotaDelta).NBytes @@ -482,7 +482,7 @@ proc putLeafBlockMetaImpl( trace "Put or update leaf and block metadata", treeCid, recordsCount = updates.len if err =? ( - await self.metaDs.tryPutAtomic(updates, maxRetries = 10, putLeafAndBlockMetaAtomic) + await self.metaDs.tryPutAtomic(updates, maxRetries = 30, putLeafAndBlockMetaAtomic) ).errorOption: trace "Unable to put or update leaf and block metadata", error = err.msg return failure(err) diff --git a/archivist/streams/storestream.nim b/archivist/streams/storestream.nim index 61c8d5a5a..d7b266ef9 100644 --- a/archivist/streams/storestream.nim +++ b/archivist/streams/storestream.nim @@ -30,6 +30,12 @@ logScope: const StoreStreamTrackerName* = "StoreStream" +# A block batch may miss blocks under load (a peer drops a want-batch, or a +# concurrent store write retries out). Retry the fetch before aborting the +# stream - one transient miss should not kill a whole download. +const DefaultStreamRetries* = 5 +const DefaultStreamRetryDelay = 50.millis + type # Make SeekableStream from a sequence of blocks stored in Manifest # (only original file data - see StoreStream.size) @@ -100,11 +106,33 @@ method readOnce*( trace "No blocks returned from store!" raise newLPStreamReadError(newException(IOError, "No blocks returned from store!")) - # Build a lookup table from block CID to block data for ordered copying - # We copy block by block in index order using single-block getBlock fallback - # if a block is missing from the batch result. + # Build a lookup table from block index to block data, retrying any indices + # that came back missing (transient fetch/store misses under load). + var blocksMap = blocks.toTable + var missing: seq[Natural] + for idx in indices: + if not blocksMap.hasKey(idx): + missing.add(idx) + + for attempt in 1 .. DefaultStreamRetries: + if missing.len == 0: + break + trace "Retrying missing blocks", missing = missing.len, attempt + without retried =? (await self.store.getBlocks(treeCid, missing)).tryGet.catch, err: + trace "Unable to retry blocks from store", err = err.msg + break + for (idx, blk) in retried: + blocksMap[idx] = blk + var stillMissing: seq[Natural] + for idx in missing: + if not blocksMap.hasKey(idx): + stillMissing.add(idx) + missing = stillMissing + if missing.len > 0 and attempt < DefaultStreamRetries: + await sleepAsync(DefaultStreamRetryDelay) + + # Copy block by block in index order var read = 0 - let blocksMap = blocks.toTable for idx in indices: if self.atEof or read >= nbytes: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index b8a470b11..249acb286 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -153,13 +153,22 @@ if [[ "${ARCHIVIST_PROVER}" == "true" && -z "${SKIP_DOWNLOAD_CIRCUIT}" ]]; then export ARCHIVIST_CIRCUIT_DIR="${ARCHIVIST_DATA_DIR}/circuits" fi - # Download circuit + # Download circuit - retry until the RPC node is reachable. The dist-test + # harness treats a single failure as a pod crash. mkdir -p "${ARCHIVIST_CIRCUIT_DIR}" chmod 700 "${ARCHIVIST_CIRCUIT_DIR}" download="cirdl ${ARCHIVIST_CIRCUIT_DIR} ${ARCHIVIST_ETH_PROVIDER} ${ARCHIVIST_MARKETPLACE_ADDRESS}" echo "${download}" - eval "${download}" - [[ $? -ne 0 ]] && { echo "Failed to download circuit files"; exit 1; } + n=0 + until eval "${download}"; do + n=$((n + 1)) + if [ $n -ge 10 ]; then + echo "Failed to download circuit files" + exit 1 + fi + echo "Circuit download attempt $n failed, retrying in 10s..." + sleep 10 + done fi # Show diff --git a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim index 91a5fe09e..9258144c8 100644 --- a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim @@ -220,3 +220,95 @@ asyncchecksuite "Test Discovery Engine": reqs.complete() await discoveryEngine.stop() + + test "Should back off failed discovery requests": + var + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + discoveryEngine = DiscoveryEngine.new( + localStore, + peerStore, + network, + blockDiscovery, + pendingBlocks, + discoveryLoopSleep = 100.millis, + ) + calls = 0 + + blockDiscovery.findBlockProvidersHandler = proc( + d: MockDiscovery, cid: Cid + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + calls.inc + # empty result is a failed discovery + + pendingBlocks.getPeerForBlock = alwaysRequeue(30.seconds) + await pendingBlocks.start() + discard pendingBlocks.getWantHandle(blocks[0].cid) + check toSeq(pendingBlocks.wantListBlockCids).len == 1 + + await discoveryEngine.start() + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 1 + + # Backoff (3s) has elapsed, the re-queued want is processed again + await sleepAsync(2.seconds + 300.millis) + check calls == 2 + + # Backoff doubles (3s -> 6s), so the third call must not happen yet + await sleepAsync(1.seconds) + check calls == 2 + + await discoveryEngine.stop() + await pendingBlocks.stop() + + test "Should rediscover immediately once a block leaves the want list": + var + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + discoveryEngine = DiscoveryEngine.new( + localStore, + peerStore, + network, + blockDiscovery, + pendingBlocks, + discoveryLoopSleep = 100.millis, + ) + calls = 0 + + blockDiscovery.findBlockProvidersHandler = proc( + d: MockDiscovery, cid: Cid + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + calls.inc + # empty result is a failed discovery + + pendingBlocks.getPeerForBlock = alwaysRequeue(30.seconds) + await pendingBlocks.start() + discard pendingBlocks.getWantHandle(blocks[0].cid) + await discoveryEngine.start() + + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 1 + + # Second failure grows the backoff to 6s + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(3.seconds + 300.millis) + check calls == 2 + + # Resolve the block - it leaves the want list; its backoff must be pruned + await pendingBlocks.resolve(BlockAddress.init(blocks[0].cid), blocks[0]) + await sleepAsync(500.millis) # prune runs every discoveryLoopSleep (100ms) + + # Re-request the same cid - discovery must fire without the stale 6s wait + discard pendingBlocks.getWantHandle(blocks[0].cid) + discoveryEngine.queueFindBlocksReq(@[blocks[0].cid]) + await sleepAsync(1.seconds) + check calls == 3 + + await discoveryEngine.stop() + await pendingBlocks.stop() diff --git a/tests/archivist/blockexchange/engine/testadvertiser.nim b/tests/archivist/blockexchange/engine/testadvertiser.nim index aac3344d2..3ddeae296 100644 --- a/tests/archivist/blockexchange/engine/testadvertiser.nim +++ b/tests/archivist/blockexchange/engine/testadvertiser.nim @@ -110,3 +110,27 @@ asyncchecksuite "Advertiser": check: localStore.onBlockStored.isNone() + + test "Should keep retrying while the routing table is below the minimum": + await advertiser.stop() + blockDiscovery.nodesDiscoveredHandler = proc(d: MockDiscovery): int = + 2 + advertiser = Advertiser.new( + localStore, + blockDiscovery, + minAdvertisePeers = 16, + advertiseRetrySleep = 500.millis, + ) + await advertiser.start() + + (await localStore.putBlock(manifestBlk)).tryGet() + + await sleepAsync(2500.millis) # several retry rounds at 500ms cadence + check advertised.len >= 6 # initial 2 (manifest + tree) + retries + + await sleepAsync(700.millis) # let any in-flight retry round drain + let count = advertised.len + blockDiscovery.nodesDiscoveredHandler = proc(d: MockDiscovery): int = + 16 + await sleepAsync(1500.millis) + check advertised.len == count # healthy table: retry cycle stops diff --git a/tests/archivist/blockexchange/engine/testengine.nim b/tests/archivist/blockexchange/engine/testengine.nim index ca1fcb4dc..1b97a891f 100644 --- a/tests/archivist/blockexchange/engine/testengine.nim +++ b/tests/archivist/blockexchange/engine/testengine.nim @@ -269,6 +269,122 @@ asyncchecksuite "NetworkStore engine handlers": check archivist_block_exchange_want_have_entries_received.value() == beforeEntries + blocks.len.float64 + test "Should rotate refresh targets past the query quarantine": + # Regression: the refresh fallback always asked the same aggregate top-K; + # a fresh provider with zero deliveries was never asked, never reported + # presence, and was never fetched from. Quarantine makes selection rotate. + let + testPeerStore = PeerCtxStore.new() + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, network, testDiscovery, testAdvertiser, testPeerStore, pendingBlocks + ) + + var peerIds: seq[PeerId] + for i in 0 ..< 4: + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + id = PeerId.init(key.getPublicKey().tryGet()).tryGet() + peerIds.add(id) + let ctx = BlockExcPeerCtx.new(id) + testPeerStore.add(ctx) + if i < 3: + for _ in 0 ..< 2: + ctx.ensureScoreFor(blocks[0].cid).recordDelivery(1024, 10.0) + + # Round 1: the three delivery-scored peers take all top-K slots. + let first = testEngine.topPeersByAggregate(DefaultWantHaveTopK) + check first.len == 3 + for i in 0 ..< 3: + check peerIds[i] in first.mapIt(it.id) + check peerIds[3] notin first.mapIt(it.id) + + # Quarantine them - the zero-score peer rotates in. + for peer in first: + peer.markQueried() + let second = testEngine.topPeersByAggregate(DefaultWantHaveTopK) + check peerIds[3] in second.mapIt(it.id) + + # Quarantine everyone - nothing left to refresh. + for peer in second: + peer.markQueried() + check testEngine.topPeersByAggregate(DefaultWantHaveTopK).len == 0 + + test "Should randomize equal-score peers in the aggregate ranking": + # Equal-score peers (e.g. all at ColdStartScore) must not pin one top-K + # forever - a stable store order would starve the tail of the tie group. + let + testPeerStore = PeerCtxStore.new() + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, network, testDiscovery, testAdvertiser, testPeerStore, pendingBlocks + ) + + for _ in 0 ..< 4: + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + id = PeerId.init(key.getPublicKey().tryGet()).tryGet() + testPeerStore.add(BlockExcPeerCtx.new(id)) + + var distinctTops: HashSet[string] + for _ in 0 ..< 200: + distinctTops.incl($testEngine.topPeersByAggregate(2).mapIt(it.id)) + + check distinctTops.len > 1 # equal-score peers never pin one top-K + + test "Should still refresh quarantined peers when there are unsent wants": + # Regression pin: the query quarantine must not suppress WantHaves for + # addresses a peer has not been asked about yet. The connect-time Full + # send quarantines the only peer, and a later request for a new block + # must still reach it (decideSend would emit a Delta for the new wants). + var asked = newSeq[PeerId]() + + proc sendWantList( + id: PeerId, + addresses: seq[BlockAddress], + priority: int32 = 0, + cancel: bool = false, + wantType: WantType = WantType.WantHave, + full: bool = false, + sendDontHave: bool = false, + ): Future[?!void] {.async: (raises: [CancelledError]).} = + asked.add(id) + success() + + let + testPeerStore = PeerCtxStore.new() + testNetwork = + BlockExcNetwork(request: BlockExcRequest(sendWantList: sendWantList)) + testDiscovery = DiscoveryEngine.new( + localStore, testPeerStore, network, blockDiscovery, pendingBlocks + ) + testAdvertiser = Advertiser.new(localStore, blockDiscovery) + testEngine = BlockExcEngine.new( + localStore, testNetwork, testDiscovery, testAdvertiser, testPeerStore, + pendingBlocks, + ) + + # A fresh peer with no wants asked yet, quarantined. + let + key = PrivateKey.random(libp2p_rng.newBearSslRng(rng)).tryGet() + freshId = PeerId.init(key.getPublicKey().tryGet()).tryGet() + let fresh = BlockExcPeerCtx.new(freshId) + testPeerStore.add(fresh) + fresh.markQueried() + + let pending = testEngine.requestDelivery(BlockAddress.init(blocks[0].cid)).tryGet() + await sleepAsync(200.millis) + check asked.len > 0 # unsent wants override the quarantine - WantHave sent + await pending.cancelAndWait() + expect CancelledError: + discard await pending + test "Should handle want list - `dont-have`": let wantList = makeWantList(blocks.mapIt(it.cid), sendDontHave = true) var diff --git a/tests/archivist/blockexchange/engine/testpeerscoring.nim b/tests/archivist/blockexchange/engine/testpeerscoring.nim index 2cfc79dd2..afc52e278 100644 --- a/tests/archivist/blockexchange/engine/testpeerscoring.nim +++ b/tests/archivist/blockexchange/engine/testpeerscoring.nim @@ -93,3 +93,19 @@ suite "scoredPeer selector": let ranked = rankPeersByScore(@[peer], testCid) check ranked.len == 1 check ranked[0] == peer + + test "Should baseline aggregate score at ColdStartScore for new peers": + # A peer with no score entries at all must not rank at 0.0. + check aggregateScore(makeCtx()) == ColdStartScore + + test "Should NOT count WantHave evidence in the aggregate score": + # A Have-reply creates a score entry (setPresence), but counting it + # would let a peer inflate its score by reporting Haves it never + # delivers. The entry contributes 0 until deliveries happen. + check aggregateScore(makeCtxWithScore()) == 0.0 + + test "Should rank delivered peers above the baseline in aggregate": + let busy = makeCtxWithScore() + busy.ensureScoreFor(testCid).recordDelivery(1024, 10.0) + busy.ensureScoreFor(testCid).recordDelivery(1024, 10.0) + check aggregateScore(busy) > ColdStartScore diff --git a/tests/archivist/helpers/mockdiscovery.nim b/tests/archivist/helpers/mockdiscovery.nim index 74ed64fd6..0ab0e0b72 100644 --- a/tests/archivist/helpers/mockdiscovery.nim +++ b/tests/archivist/helpers/mockdiscovery.nim @@ -11,9 +11,12 @@ import pkg/chronos import pkg/libp2p import pkg/questionable import pkg/archivist/discovery +import pkg/archivist/blockexchange/engine import pkg/contractabi/address as ca type MockDiscovery* = ref object of Discovery + nodesDiscoveredHandler*: proc(d: MockDiscovery): int {.gcsafe, raises: [].} + findBlockProvidersHandler*: proc( d: MockDiscovery, cid: Cid ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} @@ -32,6 +35,13 @@ type MockDiscovery* = ref object of Discovery proc new*(T: type MockDiscovery): MockDiscovery = MockDiscovery() +method nodesDiscovered*(d: MockDiscovery): int = + # Report a healthy table unless a test overrides it + if isNil(d.nodesDiscoveredHandler): + return DefaultMinAdvertisePeers + + d.nodesDiscoveredHandler(d) + proc findPeer*( d: Discovery, peerId: PeerId ): Future[?PeerRecord] {.async: (raises: [CancelledError]).} = diff --git a/tests/archivist/helpers/mockmarketplace.nim b/tests/archivist/helpers/mockmarketplace.nim index 707e530fb..bfb9d2c6b 100644 --- a/tests/archivist/helpers/mockmarketplace.nim +++ b/tests/archivist/helpers/mockmarketplace.nim @@ -48,6 +48,7 @@ type subscriptions: Subscriptions config*: MarketplaceConfig canReserveSlot*: bool + tokens*: UInt256 errorOnReserveSlot*: ?(ref MarketplaceError) errorOnFillSlot*: ?(ref MarketplaceError) errorOnFreeSlot*: ?(ref MarketplaceError) @@ -130,7 +131,11 @@ proc new*(_: type MockMarketplace, clock: Clock = MockClock.new()): MockMarketpl requestDurationLimit: StorageDuration.init(60 * 60 * 24 * 30.stuint(40)), ) MockMarketplace( - signer: Address.example, config: config, canReserveSlot: true, clock: clock + signer: Address.example, + config: config, + canReserveSlot: true, + tokens: high(UInt256), + clock: clock, ) method getSigner*( @@ -393,6 +398,11 @@ method canReserveSlot*( ): Future[bool] {.async.} = return marketplace.canReserveSlot +method tokensAvailable*( + marketplace: MockMarketplace +): Future[UInt256] {.async: (raises: [CancelledError, MarketplaceError]).} = + return marketplace.tokens + func setCanReserveSlot*(marketplace: MockMarketplace, canReserveSlot: bool) = marketplace.canReserveSlot = canReserveSlot @@ -520,7 +530,8 @@ method queryPastStorageRequestedEvents*( StorageRequested( requestId: request.id, ask: request.ask, - expiry: marketplace.requestExpiry[request.id], + expiry: + marketplace.requestExpiry.getOrDefault(request.id, StorageTimestamp.init(0)), ) ) @@ -532,7 +543,8 @@ method queryPastStorageRequestedEvents*( StorageRequested( requestId: request.id, ask: request.ask, - expiry: marketplace.requestExpiry[request.id], + expiry: + marketplace.requestExpiry.getOrDefault(request.id, StorageTimestamp.init(0)), ) ) diff --git a/tests/archivist/marketplace/sales/states/testerrored.nim b/tests/archivist/marketplace/sales/states/testerrored.nim index 61bc1b20a..0a1c9c213 100644 --- a/tests/archivist/marketplace/sales/states/testerrored.nim +++ b/tests/archivist/marketplace/sales/states/testerrored.nim @@ -53,7 +53,9 @@ asyncchecksuite "sales state 'errored'": test "transits to unknown state when slot is active": let me = await marketplace.getSigner() - marketplace.activeSlots[me] = @[slot.id] + marketplace.filled.add( + MockSlot(requestId: request.id, slotIndex: slotIndex, host: me) + ) let nextState = await runState() check !nextState of SaleUnknown @@ -74,5 +76,6 @@ asyncchecksuite "sales state 'errored'": check storage.deleteSlotCalls == @[(request.content.cid, slotIndex)] test "transits to error state when slots call fails": + marketplace.errorOnGetHost = some newException(MarketplaceError, "boom") let nextState = await runState() check !nextState of SaleErrored diff --git a/tests/archivist/marketplace/testpurchasing.nim b/tests/archivist/marketplace/testpurchasing.nim index 3c65191ec..55931278d 100644 --- a/tests/archivist/marketplace/testpurchasing.nim +++ b/tests/archivist/marketplace/testpurchasing.nim @@ -133,9 +133,10 @@ suite "Purchasing state machine": test "loads active purchases from marketplace": let me = await marketplace.getSigner() - let request1, request2, request3 = StorageRequest.example + var request1, request2, request3 = StorageRequest.example + request1.client = me + request2.client = me marketplace.requested = @[request1, request2, request3] - marketplace.activeRequests[me] = @[request1.id, request2.id] await purchasing.load() check isSome purchasing.getPurchase(PurchaseId(request1.id)) check isSome purchasing.getPurchase(PurchaseId(request2.id)) @@ -143,10 +144,13 @@ suite "Purchasing state machine": test "loads correct purchase.future state for purchases from marketplace": let me = await marketplace.getSigner() - let request1, request2, request3, request4, request5 = StorageRequest.example + var request1, request2, request3, request4, request5 = StorageRequest.example + request1.client = me + request2.client = me + request3.client = me + request4.client = me + request5.client = me marketplace.requested = @[request1, request2, request3, request4, request5] - marketplace.activeRequests[me] = - @[request1.id, request2.id, request3.id, request4.id, request5.id] marketplace.requestState[request1.id] = RequestState.New marketplace.requestState[request2.id] = RequestState.Started marketplace.requestState[request3.id] = RequestState.Cancelled @@ -261,11 +265,12 @@ suite "Purchasing state machine": check next.isNone test "restarts state machine when error occurs while request is active": - let request = StorageRequest.example + var request = StorageRequest.example + request.client = await marketplace.getSigner() let purchase = Purchase.new(request, marketplace, clock) let error = newException(CatchableError, "some error") - let me = await marketplace.getSigner() - marketplace.activeRequests[me] = @[request.id] + marketplace.requested.add(request) + marketplace.requestState[request.id] = RequestState.New let next = await PurchaseErrored(error: error).run(purchase) check !next of PurchaseUnknown