Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions archivist.nim
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ when isMainModule:
when defined(posix):
import system/ansi_c

when defined(linux):
# Pin the mmap threshold below the block size: block buffers stay on
# the mmap path and return memory to the OS on free. Setting it
# explicitly also disables the dynamic ratchet that demotes them.
const M_MMAP_THRESHOLD = -3
proc mallopt(param: cint, value: cint): cint {.importc, header: "<malloc.h>".}
discard mallopt(M_MMAP_THRESHOLD, 64 * 1024)

type NodeStatus {.pure.} = enum
Stopped
Stopping
Expand Down
52 changes: 32 additions & 20 deletions archivist/archivist.nim
Original file line number Diff line number Diff line change
Expand Up @@ -57,36 +57,48 @@ type

NodePrivateKey* = libp2p.PrivateKey # alias

proc connectMarketplace(s: NodeServer) {.async.} =
proc connectMarketplace(s: NodeServer) {.async: (raises: [CancelledError]).} =
let config = s.config

if config.persistence:
without ethPrivateKeyFile =? config.ethPrivateKey:
error "Persistence enabled, but no Ethereum private key was set"
quit QuitFailure

let marketplaceResult = await MarketplaceNode.connect(
ethProviderUrl = config.ethProvider,
ethPrivateKeyFile = ethPrivateKeyFile,
datastore = s.repoStore.metaDs,
storage = MarketplaceStorage.new(s.archivistNode, s.repoStore),
options = MarketplaceOptions(
marketplaceAddress: config.marketplaceAddress,
maxPriorityFeePerGas: config.maxPriorityFeePerGas,
requestCacheSize: config.marketplaceRequestCacheSize,
validationEnabled: config.validator,
validationMaxSlots: some config.validatorMaxSlots,
validationGroups: config.validatorGroups,
validationGroupIndex: some config.validatorGroupIndex,
useSystemClock: config.useSystemClock,
),
)
# The RPC node may not be reachable on the first attempt (e.g. the geth
# endpoint is still propagating after a restart); retry before giving up.
var marketplace: ?MarketplaceNode
for attempt in 1 .. config.marketplaceConnectRetries:
without res =? (
await MarketplaceNode.connect(
ethProviderUrl = config.ethProvider,
ethPrivateKeyFile = ethPrivateKeyFile,
datastore = s.repoStore.metaDs,
storage = MarketplaceStorage.new(s.archivistNode, s.repoStore),
options = MarketplaceOptions(
marketplaceAddress: config.marketplaceAddress,
maxPriorityFeePerGas: config.maxPriorityFeePerGas,
requestCacheSize: config.marketplaceRequestCacheSize,
validationEnabled: config.validator,
validationMaxSlots: some config.validatorMaxSlots,
validationGroups: config.validatorGroups,
validationGroupIndex: some config.validatorGroupIndex,
useSystemClock: config.useSystemClock,
),
)
), err:
error "Unable to connect to marketplace", error = err.msg, attempt = attempt
if attempt < config.marketplaceConnectRetries:
await sleepAsync(config.marketplaceConnectRetryDelay)
continue

marketplace = res.some
break

without marketplace =? marketplaceResult, err:
error "Unable to connect to marketplace", error = err.msg
without marketplaceNode =? marketplace:
quit QuitFailure

s.archivistNode.marketplace = marketplace
s.archivistNode.marketplace = marketplaceNode

proc start*(s: NodeServer) {.async.} =
trace "Starting node", config = $s.config
Expand Down
2 changes: 1 addition & 1 deletion archivist/archivisttypes.nim
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export tables

const
# Size of blocks for storage / network exchange,
DefaultBlockSize* = 512.KiBs
DefaultBlockSize* = 128.KiBs
DefaultCellSize* = 2.KiBs

# Proving defaults
Expand Down
16 changes: 3 additions & 13 deletions archivist/blockexchange/engine/advertiser.nim
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,8 @@ logScope:
const
DefaultConcurrentAdvertRequests = 10
DefaultAdvertiseLoopSleep = 30.minutes
DefaultMinAdvertisePeers = 16
DefaultMinAdvertisePeers* = 16
DefaultAdvertiseRetrySleep = 30.seconds
DefaultAdvertiseRetryWindow = 10.minutes

type Advertiser* = ref object of RootObj
localStore*: BlockStore # Local block store for this instance
Expand All @@ -55,8 +54,6 @@ type Advertiser* = ref object of RootObj
pendingAdvRetries: Table[Cid, Future[void]] # Delayed sparse retry tasks
minAdvertisePeers: int # Desired routing-table size before advertising
advertiseRetrySleep: Duration # Delay before retrying sparse startup advertise
advertiseRetryWindow: Duration # Startup window for sparse advertise retries
startedAt: Moment # Time advertiser started

proc addCidToQueue(b: Advertiser, cid: Cid) {.async: (raises: [CancelledError]).} =
if cid notin b.advertiseQueue:
Expand Down Expand Up @@ -113,18 +110,14 @@ proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} =

info "Exiting advertise task loop"

proc hasElapsed(since: Moment, dur: Duration): bool =
(Moment.now() - since) >= dur

proc shouldRetryAdvertise(b: Advertiser): bool =
b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers and
not hasElapsed(b.startedAt, b.advertiseRetryWindow)
b.minAdvertisePeers > 0 and b.discovery.nodesDiscovered() < b.minAdvertisePeers

proc delayedAdvertiseRetry(b: Advertiser, cid: Cid) {.async: (raises: []).} =
try:
await sleepAsync(b.advertiseRetrySleep)

if b.advertiserRunning and not hasElapsed(b.startedAt, b.advertiseRetryWindow):
if b.advertiserRunning and b.shouldRetryAdvertise():
trace "Requeueing advertisement retry",
cid, nodes = b.discovery.nodesDiscovered(), target = b.minAdvertisePeers
await b.addCidToQueue(cid)
Expand Down Expand Up @@ -192,7 +185,6 @@ proc start*(b: Advertiser) {.async: (raises: []).} =
doAssert(b.localStore.onBlockStored.isNone())
b.localStore.onBlockStored = onBlock.some

b.startedAt = Moment.now()
for i in 0 ..< b.concurrentAdvReqs:
let fut = b.processQueueLoop()
b.trackedFutures.track(fut)
Expand Down Expand Up @@ -225,7 +217,6 @@ proc new*(
advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep,
minAdvertisePeers = DefaultMinAdvertisePeers,
advertiseRetrySleep = DefaultAdvertiseRetrySleep,
advertiseRetryWindow = DefaultAdvertiseRetryWindow,
): Advertiser =
## Create a advertiser instance
##
Expand All @@ -240,5 +231,4 @@ proc new*(
advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep,
minAdvertisePeers: minAdvertisePeers,
advertiseRetrySleep: advertiseRetrySleep,
advertiseRetryWindow: advertiseRetryWindow,
)
56 changes: 50 additions & 6 deletions archivist/blockexchange/engine/discovery.nim
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
## those terms.

import std/sequtils
import std/sets

import pkg/chronos
import pkg/libp2p/cid
Expand Down Expand Up @@ -37,6 +38,10 @@ const
DefaultDiscoveryTimeout = 1.minutes
DefaultMinPeersPerBlock = 3
DefaultDiscoveryLoopSleep = 3.seconds
DefaultDiscoverBackoffBase = 3.seconds
DefaultDiscoverBackoffMax = 60.seconds
DefaultDiscoverBackoffMaxMult = 5
# base << 5 (96s) exceeds the 60s cap, so the interval saturates there

type DiscoveryEngine* = ref object of RootObj
localStore*: BlockStore # Local block store for this instance
Expand All @@ -51,19 +56,42 @@ type DiscoveryEngine* = ref object of RootObj
trackedFutures*: TrackedFutures # Tracked Discovery tasks futures
minPeersPerBlock*: int # Max number of peers with block
discoveryLoopSleep: Duration # Discovery loop sleep
discoverBackoffBase: Duration # Backoff base for failed discovery requests
discoverBackoffMax: Duration # Backoff cap for failed discovery requests
inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]]
# Inflight discovery requests
discoverBackoff: Table[Cid, tuple[due: Moment, mult: int]]
# Earliest retry time and backoff multiplier per CID

proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} =
try:
while b.discEngineRunning:
# Drop backoff state for blocks that are no longer wanted - the table
# must not grow with every cid ever queued. wantListCids covers leaf
# wants' tree cids, which also get discovery finds (searchForNewPeers).
let wanted = toHashSet(toSeq(b.pendingBlocks.wantListCids))
for cid in toSeq(b.discoverBackoff.keys):
if cid notin wanted:
b.discoverBackoff.del(cid)

for cid in toSeq(b.pendingBlocks.wantListBlockCids):
await b.discoveryQueue.put(cid)

await sleepAsync(b.discoveryLoopSleep)
except CancelledError:
trace "Discovery loop cancelled"

proc nextDiscoverBackoff(b: DiscoveryEngine, cid: Cid) {.raises: [].} =
## Advance the per-CID discovery backoff after a failed or empty find:
## increment the stored multiplier (capped at
## DefaultDiscoverBackoffMaxMult), then the next attempt is due in
## base * 2^mult, capped at discoverBackoffMax.
var mult = 0
b.discoverBackoff.withValue(cid, entry):
mult = min(entry[].mult + 1, DefaultDiscoverBackoffMaxMult)
let interval = min(b.discoverBackoffBase * (1 shl mult), b.discoverBackoffMax)
b.discoverBackoff[cid] = (due: Moment.now() + interval, mult: mult)

proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} =
## Run discovery tasks
##
Expand All @@ -77,8 +105,13 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} =
continue

let haves = b.peers.peersHave(cid)

if haves.len < b.minPeersPerBlock:
var notDue = false
b.discoverBackoff.withValue(cid, entry):
notDue = Moment.now() < entry[].due
if notDue:
continue

let request = b.discovery.find(cid)
b.inFlightDiscReqs[cid] = request
archivist_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
Expand All @@ -89,11 +122,17 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} =

if (await request.withTimeout(DefaultDiscoveryTimeout)) and
peers =? (await request).catch:
let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data)))

for i, f in dialed:
if f.failed:
await b.discovery.removeProvider(peers[i].data.peerId)
if peers.len > 0:
b.discoverBackoff.del(cid)

let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data)))
for i, f in dialed:
if f.failed:
await b.discovery.removeProvider(peers[i].data.peerId)
else:
b.nextDiscoverBackoff(cid)
else:
b.nextDiscoverBackoff(cid)
except CancelledError:
trace "Discovery task cancelled"
return
Expand Down Expand Up @@ -154,6 +193,8 @@ proc new*(
concurrentDiscReqs = DefaultConcurrentDiscRequests,
discoveryLoopSleep = DefaultDiscoveryLoopSleep,
minPeersPerBlock = DefaultMinPeersPerBlock,
discoverBackoffBase = DefaultDiscoverBackoffBase,
discoverBackoffMax = DefaultDiscoverBackoffMax,
): DiscoveryEngine =
## Create a discovery engine instance for advertising services
##
Expand All @@ -169,4 +210,7 @@ proc new*(
inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](),
discoveryLoopSleep: discoveryLoopSleep,
minPeersPerBlock: minPeersPerBlock,
discoverBackoffBase: discoverBackoffBase,
discoverBackoffMax: discoverBackoffMax,
discoverBackoff: initTable[Cid, tuple[due: Moment, mult: int]](),
)
60 changes: 41 additions & 19 deletions archivist/blockexchange/engine/engine.nim
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ const
DefaultConcurrentTasks = 30
DefaultWantBlockBatchSize = DefaultMaxBatchBlocks
# Serve slice sized so a full delivery message stays under the receiver's
# 32MB TCP window: 48 x 512KiB blocks = 24MiB.
DefaultMaxServeBatchBlocks = 48
# 32MB TCP window: 32 x 128KiB blocks = 4MiB.
DefaultMaxServeBatchBlocks = 32
DefaultWantBlockBatchTimeout = 5.millis
DiscoveryRateLimit = 3.seconds
PresenceBatchSize = DefaultMaxWantListBatchSize
Expand Down Expand Up @@ -210,9 +210,11 @@ proc refreshBlockKnowledge(
return decision.wakeAfter
of SendKind.Full:
await self.sendBatchedWantList(peer, decision.wants.toSeq, full = true)
peer.markQueried()
return Duration.none
of SendKind.Delta:
await self.sendBatchedWantList(peer, decision.wants.toSeq, full = false)
peer.markQueried()
return Duration.none

proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) =
Expand All @@ -223,6 +225,16 @@ proc minWakeHint(current: var Option[Duration], candidate: Option[Duration]) =
else:
current = wake.some

proc hasUnsentWants(self: BlockExcEngine, peer: BlockExcPeerCtx): bool =
## True when some wanted block has not been asked of this peer yet.
## The query quarantine only suppresses re-asks of the same want list -
## newly queued wants must still reach the peer (mirrors decideSend's
## delta computation without mutating the circuit state).
for address in self.pendingBlocks.wantList:
if address notin peer.blocks and address notin peer.alreadySent:
return true
false

proc searchForNewPeers(self: BlockExcEngine, cid: Cid) =
if self.lastDiscRequest + DiscoveryRateLimit < Moment.now():
archivist_block_exchange_discovery_requests.inc()
Expand All @@ -235,20 +247,24 @@ proc evictPeer(self: BlockExcEngine, peer: PeerId) {.gcsafe, async: (raises: [])
self.peers.remove(peer)

proc topPeersForCid*(self: BlockExcEngine, cid: Cid, topK: int): seq[BlockExcPeerCtx] =
let candidates = self.peers.peersHave(cid)
if candidates.len == 0:
return @[]
let ranked = rankPeersByScore(candidates, cid)
if ranked.len == 0:
return @[]
ranked[0 ..< min(topK, ranked.len)]
let ranked = rankPeersByScore(self.peers.peersHave(cid), cid)
var top: seq[BlockExcPeerCtx]
for peer in ranked:
if top.len >= topK:
break
if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer):
top.add(peer)
top

proc topPeersByAggregate*(self: BlockExcEngine, topK: int): seq[BlockExcPeerCtx] =
let allPeers = toSeq(self.peers.peers.values)
let ranked = rankPeersByAggregate(allPeers)
if ranked.len == 0:
return @[]
ranked[0 ..< min(topK, ranked.len)]
let ranked = rankPeersByAggregate(toSeq(self.peers.peers.values))
var top: seq[BlockExcPeerCtx]
for peer in ranked:
if top.len >= topK:
break
if peer.queryQuarantineLeft().isNone or self.hasUnsentWants(peer):
top.add(peer)
top

proc failBlockRequest(
self: BlockExcEngine,
Expand Down Expand Up @@ -882,11 +898,7 @@ proc new*(
candidates = self.peers.peersHave(cid)
topPeers =
if candidates.len > 0:
let ranked = rankPeersByScore(candidates, cid)
if ranked.len == 0:
@[]
else:
ranked[0 ..< min(DefaultWantHaveTopK, ranked.len)]
self.topPeersForCid(cid, DefaultWantHaveTopK)
else:
self.topPeersByAggregate(DefaultWantHaveTopK)

Expand All @@ -899,6 +911,16 @@ proc new*(
minWakeHint(minWake, fut.read())
except CatchableError:
discard
else:
# Every candidate was queried recently - wake when the first quarantine
# expires instead of re-running selection in a tight loop.
let pool =
if candidates.len > 0:
candidates
else:
toSeq(self.peers.peers.values)
for peer in pool:
minWakeHint(minWake, peer.queryQuarantineLeft())

let discoveredPeer = selectKnownPeer(address)
if not discoveredPeer.isNil:
Expand Down
4 changes: 4 additions & 0 deletions archivist/blockexchange/engine/peerscoring.nim
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC
let s = effectiveScore(peer, cid, now)
ranked.add((s, peer))

# Randomize equal-score groups (stable sort preserves shuffle order)
Rng.instance.shuffle(ranked)
ranked.sort(
proc(a, b: auto): int =
cmp(b[0], a[0])
Expand All @@ -46,6 +48,8 @@ proc rankPeersByScore*(peers: seq[BlockExcPeerCtx], cid: Cid): seq[BlockExcPeerC

proc rankPeersByAggregate*(peers: seq[BlockExcPeerCtx]): seq[BlockExcPeerCtx] =
var ranked = peers.mapIt((aggregateScore(it), it))
# Randomize equal-score groups (stable sort preserves shuffle order)
Rng.instance.shuffle(ranked)
ranked.sort(
proc(a, b: auto): int =
cmp(b[0], a[0])
Expand Down
Loading
Loading