diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 183100a84d4..63b958576ff 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -91,6 +91,23 @@ jobs: ls -la "staging/${{ matrix.rid }}/native" file "staging/${{ matrix.rid }}/native"/*.so + - name: Verify required exports + run: | + set -euo pipefail + so="staging/${{ matrix.rid }}/native/libnative_device.so" + # The C# NativeStorageDevice loader hard-probes these at device creation: ImportResolver + # binds the P/Invokes and the startup ABI probe calls NumIoContexts / QueueRunFor / + # TryCompleteMine (the affine inline-drain path is on by default). CreateWithBackend is the + # create export the managed wrapper binds; a binary missing any of these throws at server + # startup on this RID, so fail the build here instead of shipping it. + # Host nm reads the ELF dynamic symbol table for every RID (x64/arm64, glibc/musl) alike. + command -v nm >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq binutils; } + rc=0 + for sym in NativeDevice_CreateWithBackend NativeDevice_NumIoContexts NativeDevice_QueueRunFor NativeDevice_TryCompleteMine; do + if nm -D --defined-only "$so" | grep -qw "$sym"; then echo " OK $sym"; else echo " MISSING $sym"; rc=1; fi + done + [ $rc -eq 0 ] || { echo "::error::$so is missing a required NativeDevice_* export the C# loader probes at startup"; exit 1; } + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: native-${{ matrix.rid }} @@ -179,10 +196,18 @@ jobs: $dll = "staging/${{ matrix.rid }}/native/native_device.dll" $exports = & $dumpbin /exports $dll - if (-not ($exports | Select-String 'NativeDevice_CreateWithBackend')) { - throw "native_device.dll is missing NativeDevice_CreateWithBackend export" + # The C# NativeStorageDevice loader hard-probes these at device creation: ImportResolver + # binds the P/Invokes and the startup ABI probe calls NumIoContexts / QueueRunFor / + # TryCompleteMine (the affine inline-drain path is on by default). CreateWithBackend is the + # create export the managed wrapper binds; a binary missing any of these throws at server + # startup on this RID, so fail the build here instead of shipping it. + $required = @('NativeDevice_CreateWithBackend','NativeDevice_NumIoContexts','NativeDevice_QueueRunFor','NativeDevice_TryCompleteMine') + foreach ($sym in $required) { + if (-not ($exports | Select-String -SimpleMatch $sym)) { + throw "native_device.dll is missing required export $sym" + } } - Write-Host "Exports verified." + Write-Host "Exports verified: $($required -join ', ')." # Confirm the security flags in CMakeLists (/guard:cf and /GS /sdl, set alongside /Qspectre) # actually took effect in the produced binary: Control Flow Guard and the stack Security diff --git a/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs b/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs index 5dccc90990b..a1d77aa9683 100644 --- a/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs +++ b/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs @@ -15,7 +15,7 @@ public unsafe partial class ReqGen static int bitfieldOpCount = 3; readonly byte[][] buffers; - readonly List> flatRequestBuffer; + readonly List flatRequestBuffer; readonly int[] lens; readonly OpType opType; readonly bool randomGen, randomServe; @@ -67,7 +67,7 @@ public ReqGen( } buffers = new byte[NumBuffs][]; - flatRequestBuffer = flatBufferClient ? new List>() : null; + flatRequestBuffer = flatBufferClient ? new List() : null; lens = new int[NumBuffs]; BatchCount = BatchSize; this.opType = opType; @@ -114,7 +114,7 @@ public byte[] GetRequest(out int len) int offset; if (randomServe) - offset = r.Next(NumBuffs); + offset = Random.Shared.Next(NumBuffs); else offset = (Interlocked.Increment(ref seqNo) - 1) % NumBuffs; @@ -122,11 +122,11 @@ public byte[] GetRequest(out int len) return buffers[offset]; } - public List GetRequestArgs() + public string[] GetRequestArgs() { int offset; if (randomServe) - offset = r.Next(flatRequestBuffer.Count); + offset = Random.Shared.Next(flatRequestBuffer.Count); else offset = (Interlocked.Increment(ref seqNo) - 1) % flatRequestBuffer.Count; @@ -150,9 +150,7 @@ private void ConvertToSERedisInput(OpType opType) case OpType.GET: case OpType.SET: case OpType.MSET: - var buffer = buffers[i]; - flatRequestBuffer.Add(new List()); - ProcessArgs(i, buffer); + ProcessArgs(buffers[i]); break; default: Console.WriteLine($"op {opType} not supported with SERedis! Skipping conversion to SERedis input!"); @@ -161,19 +159,21 @@ private void ConvertToSERedisInput(OpType opType) } } - private void ProcessArgs(int i, byte[] buffer) + private void ProcessArgs(byte[] buffer) { fixed (byte* buf = buffer) { byte* ptr = buf; RespReadUtils.TryReadUnsignedArrayLength(out int count, ref ptr, buf + buffer.Length); - RespReadUtils.TryReadStringWithLengthHeader(out var cmd, ref ptr, buf + buffer.Length); - for (int j = 0; j < count - 1; j++) - { - RespReadUtils.TryReadStringWithLengthHeader(out var arg, ref ptr, buf + buffer.Length); - flatRequestBuffer[i].Add(arg); - } + // Keep the command token (e.g. "MSET") as element 0 so the cached array is a complete, + // ready-to-send argument list. Serve-side clients pass it straight to Execute with no + // per-request allocation, copy, or command prepend; consumers that only want the payload + // (e.g. SERedis key/value pairs) start reading at index 1. + var args = new string[count]; + for (int j = 0; j < count; j++) + RespReadUtils.TryReadStringWithLengthHeader(out args[j], ref ptr, buf + buffer.Length); + flatRequestBuffer.Add(args); } } diff --git a/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs b/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs index 01be17ad037..37720cd1d62 100644 --- a/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs +++ b/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs @@ -295,7 +295,7 @@ public void Run( rg = run_rg; else { - rg = new ReqGen(Start, opts.DbSize, TotalOps, BatchSize, opType, randomGen, randomServe, keyLen, valueLen, ttl: ttl); + rg = new ReqGen(Start, opts.DbSize, TotalOps, BatchSize, opType, randomGen, randomServe, keyLen, valueLen, flatBufferClient: (opts.Client == ClientType.SERedis || opts.Client == ClientType.GarnetClientSession), ttl: ttl); rg.Generate(); } @@ -476,11 +476,11 @@ private void GarnetClientSessionOperateThreadRunner(int NumOps, OpType opType, R Stopwatch sw = new(); sw.Start(); + // GetRequestArgs returns a shared, cached argument array that already includes the "MSET" command + // token at index 0, so it is sent directly with no per-request allocation, copy, or mutation. while (!done) { - var reqArgs = rg.GetRequestArgs(); - reqArgs.Insert(0, "MSET"); - c.Execute([.. reqArgs]); + c.Execute(rg.GetRequestArgs()); c.CompletePending(true); numReqs++; if (numReqs == maxReqs) break; @@ -511,7 +511,8 @@ private void SERedisOperateThreadRunner(int NumOps, OpType opType, ReqGen rg) while (!done) { var reqArgs = rg.GetRequestArgs(); - for (var i = 0; i < reqArgs.Count; i += 2) + // Index 0 is the command token ("MSET"); key/value pairs start at index 1. + for (var i = 1; i < reqArgs.Length; i += 2) db.StringSet(reqArgs[i], reqArgs[i + 1]); numReqs++; if (numReqs == maxReqs) break; diff --git a/benchmark/Resp.benchmark/Options.cs b/benchmark/Resp.benchmark/Options.cs index 5217aab7dec..111b622e92d 100644 --- a/benchmark/Resp.benchmark/Options.cs +++ b/benchmark/Resp.benchmark/Options.cs @@ -53,6 +53,9 @@ public partial class Options [Option('t', "threads", Separator = ',', Default = new[] { 1, 2, 4, 8, 16, 32 }, HelpText = "Number of threads (comma separated)")] public IEnumerable NumThreads { get; set; } + [Option("load-threads", Required = false, Default = 8, HelpText = "Number of threads used for the initial data load phase")] + public int LoadThreads { get; set; } + [Option('a', "auth", Required = false, Default = null, HelpText = "Authentication password")] public string Auth { get; set; } diff --git a/benchmark/Resp.benchmark/Program.cs b/benchmark/Resp.benchmark/Program.cs index 8fff7499692..4f57424eb47 100644 --- a/benchmark/Resp.benchmark/Program.cs +++ b/benchmark/Resp.benchmark/Program.cs @@ -333,7 +333,11 @@ static void RunBasicCommandsBenchmark(Options opts) var bench = new RespPerfBench(opts, 0, redis); if (!opts.SkipLoad) - bench.LoadData(keyLen: keyLen, valueLen: valueLen, numericValue: opts.Op == OpType.INCR); + { + if (opts.LoadThreads < 1) + throw new Exception($"--load-threads must be at least 1 (got {opts.LoadThreads})"); + bench.LoadData(loadDbThreads: opts.LoadThreads, keyLen: keyLen, valueLen: valueLen, numericValue: opts.Op == OpType.INCR); + } // --runtime 0 seeds the keyspace only; skip the run phase. if (opts.RunTime != 0) diff --git a/benchmark/Resp.benchmark/README.md b/benchmark/Resp.benchmark/README.md index b890051cb6c..06f29b999d9 100644 --- a/benchmark/Resp.benchmark/README.md +++ b/benchmark/Resp.benchmark/README.md @@ -36,7 +36,7 @@ Always measure on a **Release** build. `dotnet $RB --help` lists all flags. |---|---|---| | `--op` | `GET` | Op to benchmark (offline): GET, MGET, INCR, SET, ZADD, ... | | `--dbsize` | `1024` | Distinct keys (pre-loaded unless `-s`). | -| `--valuelength` | `8` | Value bytes (use `100` for KV.benchmark parity). | +| `--valuelength` | `8` | Value bytes (use `--keylength 16 --valuelength 96` = 128 B record for KV/Device parity). | | `-t` | `1,2,4,8,16,32` | Thread-count sweep (offline). | | `-b` | `4096` | Requests per pipeline (offline; dominant throughput knob, `1024` is a good default). Online forces `1`. | | `--runtime` | `15` | Seconds per cell. `0` = load only (no run). | @@ -63,22 +63,54 @@ dotnet $RB -s --op GET --dbsize 16777216 --valuelength 100 -t 1,2,4,8,16,32 -b 1 ### 2. NVMe storage-bound — reads hit real disk Tier the store with a tiny memory log so ~99.9% of a 100 M dataset is on NVMe and -every GET is a 4 KB random fetch. Use **100 M × 100 B** (smaller datasets touch few -NAND dies and understate device IOPS). +every GET is a random device fetch. Use **100 M × 128 B** records (`--keylength 16 +--valuelength 96`, matching the KV/Device benchmarks — 128 B records read over the +array's 512 B sectors). Reference host: **8×NVMe RAID-0** (`/raid`, `fio` random-read +ceiling ≈ **8.24 M IOPS at 4 K** / **8.20 M at 512 B** — the array is IOPS-bound, so +block size barely moves it); Garnet sustains **~7.3 M** end-to-end (≈ 89% of `fio`). ```bash -DATA=/mnt/nvme/garnet; mkdir -p $DATA -numactl --cpunodebind=0 --membind=0 dotnet $GS --port 6379 \ - --memory 16m --page 4m --segment 1g --index 4g --storage-tier --logdir $DATA \ - --device-type Native --device-io-backend libaio --device-throttle-limit 512 \ - --logger-level Warning & -numactl --cpunodebind=0 --membind=0 dotnet $RB --op MSET --dbsize 100000000 --valuelength 100 -t 8 -b 1024 --runtime 0 -numactl --cpunodebind=0 --membind=0 dotnet $RB -s --op GET --dbsize 100000000 --valuelength 100 -t 1,2,4,8,16,32 -b 1024 --runtime 15 +DATA=/raid/garnet; mkdir -p $DATA +# Server pinned to NUMA node 0, client driven from node 1: +numactl --cpunodebind=0 --membind=0 dotnet $GS --port 6379 --bind 127.0.0.1 \ + --memory 16m --page 4m --segment 1g --index 8g --storage-tier --logdir $DATA \ + --device-type Native --device-io-backend Libaio --logger-level Warning & +numactl --cpunodebind=1 --membind=1 dotnet $RB --op MSET --dbsize 100000000 \ + --keylength 16 --valuelength 96 --client LightClient --load-threads 32 -b 4096 --runtime 0 +numactl --cpunodebind=1 --membind=1 dotnet $RB -s --op GET --dbsize 100000000 \ + --keylength 16 --valuelength 96 --client LightClient -t 8,32,48,64 -b 1024 --runtime 12 ``` -- `--index 4g` for 100 M keys (default 128 m → 3–4× slowdown from hash chains). -- `libaio` is fastest on Linux (`uring` to compare; `Default` → RandomAccess, slower). - `--device-throttle-limit 512` is safe on fast NVMe; lower to 128 on SATA. +**Check the load before trusting the GET numbers.** A key that was never written is +answered from memory without touching the device, and a miss is ~30× cheaper than a +disk read, so a partial load silently inflates the result — an unloaded store reports +>150 M ops/sec on this host. Confirm the `MSET` step reports ~100 M ops in its +`[Total time]` line; the [generator script](scripts/nvme-raid0-matrix.sh) enforces this. + +| backend | NUMA | t=8 | t=32 | t=48 | t=64 | +|---|---|---|---|---|---| +| Libaio | srv node-0 / cli node-1 | 1.82 M | 5.70 M | 6.97 M | **7.06 M** | +| Libaio | no pin | 1.38 M | 5.09 M | 5.72 M | 5.57 M | + +> `Libaio` (the Linux default) is shown here for a quick look. For the full +> backend × pin matrix — including `Uring` on out-of-box defaults, which reaches a +> slightly higher peak — see [Sample results](#sample-results--8-nvme-ssd-raid-0) below. + +- `--index 8g` for 100 M keys (default 128 m → 3–4× slowdown from hash chains). +- Peak is in the **t=48–64** band: the RESP server's pipelined client connections drive + in-flight depth through the server's own network + completion threads, so throughput + keeps climbing well past the raw device's t=32 peak before queueing costs take over. +- **NUMA pinning matters most here** (stateful server): pinning the server to node 0 + and the client to node 1 lifts t=48 from 5.72 → 6.97 M and t=64 from 5.57 → 7.06 M. +- **`Libaio`** is the Linux default and needs no ring tuning. **`Uring`** now auto-sizes + its ring count to `min(2 × cores, 64)` — decoupled from `--device-completion-threads` — + so it is competitive with libaio out of the box; use **`--device-io-contexts N`** to set + the ring count explicitly (at or above your connection count) for very high concurrency + (see [Device Tuning](https://microsoft.github.io/garnet/docs/dev/device-tuning) and the + [Device README](../../libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md#nvme-storage-bound)). +- Capacity knobs are left at their defaults above; add `--device-completion-threads 8 + --device-throttle-limit 4096` to hand-tune. `--device-throttle-limit 4096` suits this + array; lower to 512/128 on a single/SATA disk. ### 3. Memory-device-bound — reads hit the in-RAM device @@ -94,6 +126,62 @@ LocalMemory runs). Replace the device flags in (2) with: Reference (10 M × 100 B, t=16): **~2.7 M ops/sec** at `-b 1024`, **~3.7 M** at `-b 256`. +## Sample results — 8× NVMe SSD RAID-0 + +Full **scenario 2** GET throughput matrix on **out-of-box device defaults** — only `--storage-tier` +and `--device-io-backend` are set; `--device-completion-threads`, `--device-throttle-limit`, and +`--device-io-contexts` are left at their server defaults, so this is what an operator gets with zero +device tuning. Median of 3 passes per cell. + +**Host** — 2× Intel Xeon Platinum 8480CL (56 cores × 2 threads/socket, 224 logical CPUs, 2 NUMA nodes), +~2 TB DDR5; **8× Kioxia KCM6DRUL3T84** 3.84 TB PCIe-Gen4 NVMe in Linux `md` RAID-0 (`/dev/md1`, 512 KB +chunks, ext4, ≈28 TB); Ubuntu 24.04.4 LTS, kernel 6.8.0-136, .NET 10.0.302; `fs.aio-max-nr` = 4194304. +`fio` random-read ceiling on this array: **8.24 M IOPS at 4 K** and **8.20 M IOPS at 512 B** +(32 jobs × QD64, io_uring, `O_DIRECT`, 8 files) — the array is IOPS-bound at these sizes, so the +ceiling is effectively block-size independent. + +**Workload** — 100 M × 128 B records (`--keylength 16 --valuelength 96`) tiered onto the array +(`--memory 16m --page 4m --segment 1g --index 8g`); 100% random GET; client `-b 1024`; 12 s per cell. +The generator verifies that the load actually wrote ~100 M keys before it measures, so every cell +below is storage-bound rather than partly served as in-memory misses. + +| backend | NUMA | t=8 | t=32 | t=48 | t=64 | +|---|---|---|---|---|---| +| Libaio | srv node-0 / cli node-1 | 1.82 M | 5.70 M | 6.97 M | 7.06 M | +| Libaio | no pin | 1.38 M | 5.09 M | 5.72 M | 5.57 M | +| Uring | srv node-0 / cli node-1 | 1.82 M | 6.13 M | **7.32 M** | 6.89 M | +| Uring | no pin | 1.48 M | 4.60 M | 5.55 M | 6.29 M | + +- **Peak ≈ 7.3 M ops/sec** (uring, pinned, t=48) — **~89% of the `fio` ceiling** for random reads + of the same shape (128 B records fetched over the array's 512 B sectors), driven end-to-end + through the RESP protocol and the Tsavorite pending-read path (not raw device IO). The remaining + gap to `fio` is the RESP + pending-read path: the raw device layer reaches ~8.4 M on this array. +- **The two backends peak at different thread counts.** Uring peaks at t=48 and eases off at t=64, + where added client concurrency costs more in queueing than it recovers in in-flight depth; libaio + is still climbing at t=64. Sweep the t=48–64 band rather than assuming a single best value. +- **Defaults reach the tuned peak.** Uring's smart ring-count default (`min(2 × cores, 64)` rings, + decoupled from the 4 completion threads) sizes rings to the hardware with no flags; libaio + needs no ring tuning. Explicit tuning (`--device-completion-threads 8 --device-throttle-limit 4096`, + uring `--device-io-contexts 96`) moves each cell by only a few percent on this host. +- **NUMA pinning is the largest single factor** on this dual-socket box (stateful server): e.g. uring + t=48 rises 5.55 → 7.32 M when the server is pinned to node 0 and the client to node 1. On a + single-socket host the pin / no-pin rows converge. +- Both backends reach the same ~7 M plateau once each has its required ring config — uring leads by + 5–8% at t=32–48, libaio by ~2% at t=64 — so pick either and tune the thread count (see + [Device Tuning](https://microsoft.github.io/garnet/docs/dev/device-tuning)). + +Reproduce with the checked-in generator (needs Release builds of `GarnetServer` + `Resp.benchmark`, +`numactl`, and an NVMe / O_DIRECT mount): + +```bash +DATA=/mnt/nvme/garnet benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh +``` + +It sweeps both backends × pin/no-pin × threads, takes the median of `PASSES` (default 3) per cell, and +prints the Markdown table above. Override `DATA`, `THREADS`, `PASSES`, `RUNTIME`, or set +`CT` / `THROTTLE` / `URING_IOCTX` to run the explicitly tuned configuration instead of the defaults. +Generator: [`scripts/nvme-raid0-matrix.sh`](scripts/nvme-raid0-matrix.sh). + ## Offline variations ```bash diff --git a/benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh b/benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh new file mode 100755 index 00000000000..2b5c450c64f --- /dev/null +++ b/benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# +# nvme-raid0-matrix.sh — reproducible RESP GET throughput matrix for the NVMe +# storage-bound scenario (Resp.benchmark README scenario 2). +# +# Sweeps {Libaio, Uring} x {NUMA-pinned, no-pin} x thread-count, driving a real +# GarnetServer whose 100 M x 128 B dataset is tiered onto an NVMe device so every +# GET is a random disk read. Prints a Markdown table of median-of-N throughput. +# +# This is the generator behind the "Sample results — 8x NVMe SSD RAID-0" table in +# the Resp.benchmark README. Re-run it to regression-check device-serving perf. +# +# Requirements: Release builds of GarnetServer + Resp.benchmark, numactl, an NVMe +# (or other O_DIRECT-capable) mount for the tiered log, and Linux (Native device). +# +# Usage: +# benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh +# +# Override any of the environment variables below, e.g.: +# DATA=/mnt/nvme/garnet THREADS="8 32 64" PASSES=3 \ +# benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh +# +set -uo pipefail + +# ---- configuration (override via environment) -------------------------------- +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +TFM="${TFM:-net10.0}" +GS="${GS:-$ROOT/main/GarnetServer/bin/Release/$TFM/GarnetServer.dll}" +RB="${RB:-$ROOT/benchmark/Resp.benchmark/bin/Release/$TFM/Resp.benchmark.dll}" +DATA="${DATA:-/raid/garnet-nvme-matrix}" # tiered-log dir on an NVMe mount +PORT="${PORT:-6379}" +DBSIZE="${DBSIZE:-100000000}" # 100 M keys +KEYLEN="${KEYLEN:-16}"; VALLEN="${VALLEN:-96}" # 128 B records +REQB="${REQB:-1024}" # client pipeline depth (-b) +RUNTIME="${RUNTIME:-12}" # seconds per GET cell +THREADS="${THREADS:-8 32 48 64}" # client thread-count sweep +PASSES="${PASSES:-3}" # passes per cell (median taken) +BACKENDS="${BACKENDS:-Libaio Uring}" +PINMODES="${PINMODES:-pin nopin}" +CT="${CT:-}" # --device-completion-threads (empty = server default 4) +THROTTLE="${THROTTLE:-}" # --device-throttle-limit (empty = server default 4096) +URING_IOCTX="${URING_IOCTX:-}" # Uring --device-io-contexts (empty = smart default, ~64 rings) +SRVNODE="${SRVNODE:-0}"; CLINODE="${CLINODE:-1}" # NUMA nodes for pinned mode +OUT="${OUT:-/tmp/nvme-raid0-matrix.tsv}" +SRV_LOG="${SRV_LOG:-/tmp/nvme-matrix-srv.log}" +LOAD_LOG="${LOAD_LOG:-/tmp/nvme-matrix-load.log}" + +# ---- helpers ----------------------------------------------------------------- +have() { command -v "$1" >/dev/null 2>&1; } +NUMACTL="numactl"; have numactl || NUMACTL="" + +srv_pid() { pgrep -f "dotnet .*GarnetServer.dll .*--port $PORT" | head -1; } + +wait_ready() { # wait until the server accepts connections + for _ in $(seq 1 60); do + if have redis-cli && redis-cli -p "$PORT" PING >/dev/null 2>&1; then return 0; fi + if (exec 3<>/dev/tcp/127.0.0.1/"$PORT") 2>/dev/null; then exec 3>&- 3<&-; return 0; fi + sleep 1 + done + return 1 +} + +loaded_ops() { # ops the loader reports it pushed, or empty if the line is absent + grep -E '^\[Total time\]' "$LOAD_LOG" | tail -1 | grep -oE 'for [0-9,]+' | tr -cd '0-9' +} + +stop_srv() { # stop by real GarnetServer.dll PID (not the dotnet launcher) + local p; p="$(srv_pid)" + [ -n "$p" ] && kill "$p" 2>/dev/null + for _ in $(seq 1 30); do [ -z "$(srv_pid)" ] && break; sleep 1; done + p="$(srv_pid)"; [ -n "$p" ] && kill -9 "$p" 2>/dev/null; sleep 1 +} + +median() { # median of stdin numbers + sort -n | awk '{a[NR]=$1} END{if(NR==0){print 0;exit} m=int((NR+1)/2); + if(NR%2)printf "%.3f",a[m]; else printf "%.3f",(a[m]+a[m+1])/2}' +} + +# ---- run one (backend, pinmode) server + its thread/pass sweep --------------- +run_config() { + local backend="$1" pinmode="$2" + local srv_pin="" cli_pin="" + if [ "$pinmode" = "pin" ] && [ -n "$NUMACTL" ]; then + srv_pin="$NUMACTL --cpunodebind=$SRVNODE --membind=$SRVNODE" + cli_pin="$NUMACTL --cpunodebind=$CLINODE --membind=$CLINODE" + fi + + # Out-of-box device config: only the backend is forced. All capacity knobs + # (completion-threads, throttle-limit, io-contexts) are left at the server + # defaults unless explicitly overridden via the environment — this is the + # config an external user gets with just `--storage-tier`. Set CT/THROTTLE/ + # URING_IOCTX to reproduce the hand-tuned configuration instead. + local devflags="--device-type Native --device-io-backend $backend" + [ -n "$CT" ] && devflags="$devflags --device-completion-threads $CT" + [ -n "$THROTTLE" ] && devflags="$devflags --device-throttle-limit $THROTTLE" + [ "$backend" = "Uring" ] && [ -n "$URING_IOCTX" ] && devflags="$devflags --device-io-contexts $URING_IOCTX" + + rm -rf "$DATA"; mkdir -p "$DATA" + stop_srv + # shellcheck disable=SC2086 + $srv_pin dotnet "$GS" --port "$PORT" --bind 127.0.0.1 \ + --memory 16m --page 4m --segment 1g --index 8g --storage-tier --logdir "$DATA" \ + $devflags --logger-level Warning >"$SRV_LOG" 2>&1 & + if ! wait_ready; then echo "!! server failed to start ($backend/$pinmode)"; cat "$SRV_LOG"; return 1; fi + + # load the 100 M dataset (writes tier to the device; no run phase) + # shellcheck disable=SC2086 + if ! $cli_pin dotnet "$RB" --port "$PORT" --op MSET --dbsize "$DBSIZE" --keylength "$KEYLEN" --valuelength "$VALLEN" \ + --client LightClient --load-threads 32 -b 4096 --runtime 0 >"$LOAD_LOG" 2>&1; then + echo "!! dataset load failed ($backend/$pinmode)"; tail -20 "$LOAD_LOG"; stop_srv; return 1 + fi + + # Guard against a silently short load: a key that was never written is answered from + # memory with no device IO, so a partial load publishes a number that is not + # storage-bound. The inflation is steep because a miss is ~30x cheaper than a disk + # read, so the threshold is tight: at 99% the reported figure is within ~1% of the + # fully-loaded value, whereas a 10% shortfall would overstate it by ~11%. The check + # uses the op count the loader reports rather than DBSIZE, which scans the whole index + # on a 100 M-key store and takes far longer than a startup probe should. + local floor=$((DBSIZE / 100 * 99)) + local loaded; loaded="$(loaded_ops)" + if [ -z "$loaded" ] || [ "$loaded" -lt "$floor" ]; then + echo "!! dataset load short ($backend/$pinmode): loaded ${loaded:-unknown} ops, expected >= $floor" + stop_srv; return 1 + fi + + for t in $THREADS; do + local vals=() + for _ in $(seq 1 "$PASSES"); do + # shellcheck disable=SC2086 + local out; out="$($cli_pin dotnet "$RB" -s --port "$PORT" --op GET --dbsize "$DBSIZE" \ + --keylength "$KEYLEN" --valuelength "$VALLEN" --client LightClient \ + -t "$t" -b "$REQB" --runtime "$RUNTIME" 2>/dev/null)" + # Throughput line: "[Throughput]: 1,843,118.13 ops/sec" (comma-separated) + local tp; tp="$(echo "$out" | grep -iE '^\[Throughput\]' | tr -d ',' | grep -oE '[0-9]+\.[0-9]+' | head -1)" + [ -z "$tp" ] && tp=0 + # convert ops/sec -> Mops/sec + local m; m="$(awk -v x="$tp" 'BEGIN{printf "%.3f", x/1000000}')" + vals+=("$m") + printf '%s\t%s\t%s\t%s\n' "$backend" "$pinmode" "$t" "$m" >> "$OUT" + done + local med; med="$(printf '%s\n' "${vals[@]}" | median)" + printf ' %-7s %-5s t=%-3s -> median %s M/s (passes: %s)\n' "$backend" "$pinmode" "$t" "$med" "${vals[*]}" + done + stop_srv + rm -rf "$DATA" +} + +# ---- main -------------------------------------------------------------------- +[ -f "$GS" ] || { echo "GarnetServer.dll not found: $GS (build Release first)"; exit 1; } +[ -f "$RB" ] || { echo "Resp.benchmark.dll not found: $RB (build Release first)"; exit 1; } +: > "$OUT" +echo "== NVMe RAID-0 RESP GET matrix ==" +echo " GS=$GS" +echo " RB=$RB" +echo " DATA=$DATA dbsize=$DBSIZE record=$((KEYLEN+VALLEN))B reqb=$REQB runtime=${RUNTIME}s passes=$PASSES" +echo " ct=${CT:-default} throttle=${THROTTLE:-default} uring-io-contexts=${URING_IOCTX:-default(smart)} threads='$THREADS'" +echo + +for backend in $BACKENDS; do + for pinmode in $PINMODES; do + echo "-- $backend / $pinmode --" + run_config "$backend" "$pinmode" + echo + done +done + +# ---- emit the Markdown table ------------------------------------------------- +echo "==== Markdown table (median-of-$PASSES, Mops/sec) ====" +{ + printf '| backend | NUMA |'; for t in $THREADS; do printf ' t=%s |' "$t"; done; printf '\n' + printf '|---|---|'; for _ in $THREADS; do printf '%s' '---|'; done; printf '\n' + for backend in $BACKENDS; do + for pinmode in $PINMODES; do + local_label="no pin"; [ "$pinmode" = "pin" ] && local_label="srv node-$SRVNODE / cli node-$CLINODE" + printf '| %s | %s |' "$backend" "$local_label" + for t in $THREADS; do + med="$(awk -F'\t' -v b="$backend" -v p="$pinmode" -v t="$t" \ + '$1==b&&$2==p&&$3==t{print $4}' "$OUT" | median)" + printf ' %s M |' "$med" + done + printf '\n' + done + done +} | tee /tmp/nvme-raid0-matrix.md +echo +echo "raw samples: $OUT ; table: /tmp/nvme-raid0-matrix.md" diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 7893d5fb817..5126af8e42a 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -520,9 +520,28 @@ internal sealed class Options : ICloneable public int? DeviceCompletionThreads { get; set; } [IntRangeValidation(0, 65536)] - [Option("device-throttle-limit", Required = false, HelpText = "Per-device max number of in-flight IOs (IDevice.ThrottleLimit). 0 = use the device's built-in default (120 for the in-box Tsavorite devices). Raising this lets disk-bound workloads keep the queue depth high enough to saturate fast NVMe / io_uring backends. For DeviceType=LocalMemory (which has no device-wide throttle) this instead sets the per-ring in-flight capacity, rounded up to a power of two.")] + [Option("device-throttle-limit", Required = false, HelpText = "Per-device max number of in-flight IOs (IDevice.ThrottleLimit). 0 = use the device's built-in default: 4096 for the Native device (deep NVMe / io_uring queues), 120 for the managed in-box devices. Raising this lets disk-bound workloads keep the queue depth high enough to saturate fast NVMe / io_uring backends. For DeviceType=LocalMemory (which has no device-wide throttle) this instead sets the per-ring in-flight capacity, rounded up to a power of two.")] public int? DeviceThrottleLimit { get; set; } + [IntRangeValidation(0, 4096)] + [Option("device-io-contexts", Required = false, HelpText = "Linux-only, DeviceType=Native: number of independent kernel io_contexts / io_uring rings (ring COUNT), decoupled from --device-completion-threads. Critical for io_uring: set at or above submitter concurrency (roughly your connection count) so each submitter owns a ring and io_submit is contention-free; too few rings serialize submitters on a per-ring lock and cost up to ~3x. libaio is largely indifferent. 0 = device default.")] + public int? DeviceIoContexts { get; set; } + + [IntRangeValidation(0, 32768)] + [Option("device-queue-depth", Required = false, HelpText = "Linux-only, DeviceType=Native: per-ring kernel submission depth (maxEvents for io_uring_queue_init / libaio io_setup). Orthogonal to --device-io-contexts (ring count) and --device-throttle-limit (aggregate in-flight). 0 = device default. Note: for libaio, io-contexts x queue-depth is drawn from the global fs.aio-max-nr budget.")] + public int? DeviceQueueDepth { get; set; } + + [Option("device-uring-sqpoll", Required = false, HelpText = "Linux-only, DeviceType=Native + --device-io-backend uring: enable io_uring SQPOLL (IORING_SETUP_SQPOLL) so a kernel thread polls the submission queue and submissions are syscall-free. Each ring gets its own poll thread (no IORING_SETUP_ATTACH_WQ) so submission stays parallel across rings. Ignored for libaio. Off by default (opt-in). NOTE: busy-polling kernel threads consume CPU; for high-IOPS multi-ring serving benchmark it against the default per-submit path.")] + public bool? DeviceUringSqPoll { get; set; } + + [IntRangeValidation(0, 600000)] + [Option("device-uring-sqpoll-idle-ms", Required = false, HelpText = "Linux-only, DeviceType=Native + --device-io-backend uring: io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle): how long the kernel poll thread spins after the last submit before parking. 0 = native default (10s). Only meaningful with --device-uring-sqpoll.")] + public int? DeviceUringSqPollIdleMs { get; set; } + + [IntRangeValidation(1, 4096)] + [Option("device-aio-max-devices", Required = false, HelpText = "Linux-only, DeviceType=Native (libaio): target number of Native devices to fit within the machine-global fs.aio-max-nr libaio budget (default 32). libaio io_setup permanently reserves io-contexts x queue-depth events from that global budget per device, so the default per-device reservation is capped at fs.aio-max-nr / this, keeping at least this many devices creatable regardless of --device-completion-threads / --device-throttle-limit. The cap cannot go below one event per ring, so an explicit --device-io-contexts above the per-device share still exceeds it (warned). Raise fs.aio-max-nr, or lower this, to give each serving device a deeper reservation. Ignored for io_uring (no global budget) and non-Linux.")] + public int? DeviceAioMaxDevices { get; set; } + [Option("reviv-bin-record-sizes", Separator = ',', Required = false, HelpText = "#,#,...,#: For the main store, the sizes of records in each revivification bin, in order of increasing size." + " Supersedes the default --reviv; cannot be used with --reviv-in-chain-only")] @@ -953,10 +972,17 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno DeviceFactoryCreator = deviceType == DeviceType.AzureStorage ? azureFactoryCreator() : new LocalStorageNamedDeviceFactoryCreator( deviceType: deviceType, - ioBackend: DeviceIoBackend ?? NativeStorageDevice.IoBackend.Default, numCompletionThreads: DeviceCompletionThreads ?? 4, throttleLimit: DeviceThrottleLimit is > 0 ? DeviceThrottleLimit : null, - logger: logger), + logger: logger, + nativeDeviceOptions: new NativeDeviceOptions + { + IoBackend = DeviceIoBackend ?? NativeStorageDevice.IoBackend.Default, + NumIoContexts = DeviceIoContexts ?? 0, + QueueDepth = DeviceQueueDepth ?? 0, + UringSqPoll = DeviceUringSqPoll ?? false, + UringSqPollIdleMs = DeviceUringSqPollIdleMs ?? 0, + }), CheckpointThrottleFlushDelayMs = CheckpointThrottleFlushDelayMs, EnableScatterGatherGet = EnableScatterGatherGet.GetValueOrDefault(true), ReplicaSyncDelayMs = ReplicaSyncDelayMs, @@ -976,6 +1002,11 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno DeviceIoBackend = DeviceIoBackend ?? NativeStorageDevice.IoBackend.Default, DeviceCompletionThreads = DeviceCompletionThreads ?? 4, DeviceThrottleLimit = DeviceThrottleLimit ?? 0, + DeviceIoContexts = DeviceIoContexts ?? 0, + DeviceQueueDepth = DeviceQueueDepth ?? 0, + DeviceUringSqPoll = DeviceUringSqPoll ?? false, + DeviceUringSqPollIdleMs = DeviceUringSqPollIdleMs ?? 0, + DeviceAioMaxDevices = DeviceAioMaxDevices ?? 32, ObjectScanCountLimit = ObjectScanCountLimit, RevivBinRecordSizes = revivBinRecordSizes, RevivBinRecordCounts = revivBinRecordCounts, diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index c2f6c180af6..7d72699d3c5 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -391,13 +391,50 @@ "DeviceCompletionThreads" : 4, /* Per-device max number of in-flight IOs (IDevice.ThrottleLimit). */ - /* 0 = use the device's built-in default (120 for the in-box Tsavorite devices). */ + /* 0 = use the device's built-in default: 4096 for the Native device (deep NVMe / */ + /* io_uring queues), 120 for the managed in-box devices. */ /* Raising this lets disk-bound workloads keep the queue depth high enough to saturate */ /* fast NVMe / io_uring backends. */ /* For DeviceType=LocalMemory (no device-wide throttle) it instead sets the per-ring */ /* in-flight capacity, rounded up to a power of two. */ "DeviceThrottleLimit" : 0, + /* Linux-only, DeviceType=Native: number of independent kernel io_contexts / */ + /* io_uring rings (ring COUNT), decoupled from DeviceCompletionThreads. Critical for */ + /* io_uring: set at or above submitter concurrency (~ connection count) so each */ + /* submitter owns a ring and io_submit is contention-free; too few rings serialize */ + /* submitters on a per-ring lock (~3x slower). libaio is largely indifferent. */ + /* 0 = device default. */ + "DeviceIoContexts" : 0, + + /* Linux-only, DeviceType=Native: per-ring kernel submission depth (maxEvents for */ + /* io_uring_queue_init / libaio io_setup). Orthogonal to DeviceIoContexts (ring count) */ + /* and DeviceThrottleLimit (aggregate in-flight). 0 = device default. Note: for libaio */ + /* io-contexts x queue-depth is drawn from the global fs.aio-max-nr budget. */ + "DeviceQueueDepth" : 0, + + /* Linux-only, DeviceType=Native + DeviceIoBackend=Uring: enable io_uring SQPOLL */ + /* (IORING_SETUP_SQPOLL) so a kernel thread polls the submission queue and submissions */ + /* are syscall-free. Each ring gets its own poll thread (no IORING_SETUP_ATTACH_WQ) so */ + /* submission stays parallel across rings. Ignored for libaio. false = disabled */ + /* (opt-in). NOTE: busy-polling kernel threads consume CPU - for high-IOPS multi-ring */ + /* serving benchmark against the default per-submit path. */ + "DeviceUringSqPoll" : false, + + /* Linux-only, DeviceType=Native + DeviceIoBackend=Uring: io_uring SQPOLL poll-thread */ + /* idle window in milliseconds (sq_thread_idle): how long the kernel poll thread spins */ + /* after the last submit before parking. 0 = native default (10s). Only meaningful when */ + /* DeviceUringSqPoll is true. */ + "DeviceUringSqPollIdleMs" : 0, + + /* Linux-only, DeviceType=Native (libaio): target number of Native devices to fit */ + /* within the machine-global fs.aio-max-nr libaio budget (default 32). libaio io_setup */ + /* permanently reserves io-contexts x queue-depth events from that global budget per */ + /* device, so the default per-device reservation is capped at fs.aio-max-nr / this. */ + /* Raise fs.aio-max-nr, or lower this, for a deeper per-device reservation. Ignored for */ + /* io_uring (no global budget) and non-Linux. */ + "DeviceAioMaxDevices" : 32, + /* #,#,...,#: For the main store, the sizes of records in each revivification bin, in order of increasing size. Supersedes the default --enable-reviv; cannot be used with --reviv-in-chain-only */ "RevivBinRecordSizes" : null, diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index 04a191ca22e..05751ef2e7c 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -461,6 +461,52 @@ public class GarnetServerOptions : ServerOptions /// public int DeviceThrottleLimit = 0; + /// + /// For DeviceType.Native on Linux: number of independent kernel io_contexts / io_uring rings + /// (the ring COUNT), decoupled from the completion drainers. This is the critical io_uring + /// lever: set it at or above submitter concurrency so each submitter owns a ring and io_submit + /// is contention-free (few rings + many submitters serialize on the per-ring lock). libaio is + /// largely indifferent (its kernel io_context mutex is cheap). 0 means "use the device default". + /// + public int DeviceIoContexts = 0; + + /// + /// For DeviceType.Native on Linux: per-ring kernel submission depth D (maxEvents passed to + /// io_uring_queue_init / libaio io_setup). Orthogonal to (ring + /// count) and (aggregate in-flight). 0 means "use the device + /// default". libaio draws io-contexts * queue-depth from the global fs.aio-max-nr budget. + /// + public int DeviceQueueDepth = 0; + + /// + /// For DeviceType.Native on Linux with the io_uring backend ( = Uring): + /// enable io_uring SQPOLL (IORING_SETUP_SQPOLL) so a kernel thread polls the submission queue and + /// submissions are syscall-free. Each ring gets its own poll thread (no IORING_SETUP_ATTACH_WQ) so + /// submission stays parallel across rings. Ignored for libaio / on Windows. Off by default (opt-in). + /// + public bool DeviceUringSqPoll = false; + + /// + /// io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle): how long the kernel + /// poll thread spins after the last submit before parking. Only meaningful when + /// is true; 0 means "use the native default". + /// + public int DeviceUringSqPollIdleMs = 0; + + /// + /// For DeviceType.Native on Linux (libaio): target number of Native device instances the process is + /// provisioned to coexist within the machine-global fs.aio-max-nr libaio event budget. libaio + /// io_setup permanently reserves io-contexts * queue-depth events from that global budget at + /// device creation, so a default reservation ceiling of fs.aio-max-nr / this is applied + /// per-device, guaranteeing at least this many devices can always be created regardless of + /// / . Because the budget is + /// machine-global, this is a process-wide setting (applied to + /// in ) that also covers + /// devices created outside the serving factory (cluster auxiliary logs, AOF). Default 32; ignored for + /// io_uring (no global budget) and non-Linux. + /// + public int DeviceAioMaxDevices = 32; + /// /// Limit of items to return in one iteration of *SCAN command /// @@ -680,6 +726,11 @@ public GarnetServerOptions(ILogger logger = null) : base(logger) /// public void Initialize(ILoggerFactory loggerFactory = null) { + // fs.aio-max-nr is a machine-global libaio event budget shared by every device in the process, so + // the per-device io_setup reservation ceiling is a process-wide policy rather than per-device config. + // Apply it once here (Initialize runs before any serving / cluster-auxiliary / AOF device is created). + if (DeviceAioMaxDevices > 0) + NativeStorageDevice.AioMaxDevices = DeviceAioMaxDevices; } /// @@ -769,12 +820,19 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, LightEpoch epoch, St DeviceType = Devices.GetDefaultDeviceType(); DeviceFactoryCreator ??= new LocalStorageNamedDeviceFactoryCreator( deviceType: DeviceType, - ioBackend: DeviceIoBackend, numCompletionThreads: DeviceCompletionThreads, throttleLimit: DeviceThrottleLimit > 0 ? DeviceThrottleLimit : null, - logger: logger); + logger: logger, + nativeDeviceOptions: new NativeDeviceOptions + { + IoBackend = DeviceIoBackend, + NumIoContexts = DeviceIoContexts, + QueueDepth = DeviceQueueDepth, + UringSqPoll = DeviceUringSqPoll, + UringSqPollIdleMs = DeviceUringSqPollIdleMs, + }); if (DeviceType == DeviceType.Native && OperatingSystem.IsLinux()) - logger?.LogInformation("Using device type {deviceType} (io-backend={ioBackend}, completion-threads={ct}, throttle-limit={tl})", DeviceType, DeviceIoBackend, DeviceCompletionThreads, DeviceThrottleLimit > 0 ? DeviceThrottleLimit.ToString() : "device-default"); + logger?.LogInformation("Using device type {deviceType} (io-backend={ioBackend}, completion-threads={ct}, io-contexts={ioc}, queue-depth={qd}, throttle-limit={tl}, uring-sqpoll={sqpoll})", DeviceType, DeviceIoBackend, DeviceCompletionThreads, DeviceIoContexts > 0 ? DeviceIoContexts.ToString() : "device-default", DeviceQueueDepth > 0 ? DeviceQueueDepth.ToString() : "device-default", DeviceThrottleLimit > 0 ? DeviceThrottleLimit.ToString() : "device-default", DeviceIoBackend == NativeStorageDevice.IoBackend.Uring && DeviceUringSqPoll ? "on" : "off"); else logger?.LogInformation("Using device type {deviceType} (throttle-limit={tl})", DeviceType, DeviceThrottleLimit > 0 ? DeviceThrottleLimit.ToString() : "device-default"); diff --git a/libs/storage/Tsavorite/cc/src/device/file_linux.cc b/libs/storage/Tsavorite/cc/src/device/file_linux.cc index d6ccef67d5f..2292de1e33b 100644 --- a/libs/storage/Tsavorite/cc/src/device/file_linux.cc +++ b/libs/storage/Tsavorite/cc/src/device/file_linux.cc @@ -32,6 +32,15 @@ namespace { constexpr int kSubmitYieldBudget = 16; } // anonymous namespace +namespace { +// Max completions to reap per io_getevents in the opportunistic TryComplete()/TryCompleteFor() +// poll. Reaping >1 event per syscall amortises the io_getevents call + its kernel aio-context +// ring-lock across many completions when several are ready (bursty / lower-QD completion), and is +// harmless at the saturated peak (the poll simply returns fewer than the max). Fixed at 8 +// (matching IO_BATCH_EVENTS, the dedicated-drainer reap batch). +constexpr int kTryCompleteBatchEvents = 8; +} // anonymous namespace + using namespace FASTER::core; #ifdef _DEBUG @@ -189,15 +198,17 @@ bool QueueIoHandler::TryCompleteFor(int idx) { if (ctx == 0) return false; struct timespec timeout; std::memset(&timeout, 0, sizeof(timeout)); - struct io_event events[1]; - int result = ::io_getevents(ctx, 1, 1, events, &timeout); - if(result == 1) { - io_callback_t callback = reinterpret_cast(events[0].data); - callback(ctx, events[0].obj, events[0].res, events[0].res2); - return true; - } else { - return false; + struct io_event events[kTryCompleteBatchEvents]; + // Reap up to a batch of ready completions in a single (non-blocking, timeout=0) io_getevents, + // amortising the syscall + kernel aio-context ring-lock over many events. min_nr stays 1 so a + // zeroed timeout makes this a pure poll (returns immediately with 0..max ready events). + int result = ::io_getevents(ctx, 1, kTryCompleteBatchEvents, events, &timeout); + if (result <= 0) return false; + for (int i = 0; i < result; ++i) { + io_callback_t callback = reinterpret_cast(events[i].data); + callback(ctx, events[i].obj, events[i].res, events[i].res2); } + return true; } #define IO_BATCH_EVENTS 8 /* number of events to batch up */ @@ -355,7 +366,7 @@ Status QueueFile::ScheduleOperation(FileOperationType operationType, uint8_t* bu // effectively contention-free at the kernel side. io_context_t ctx = handler_->pick_context(); - // Exactly one iocb is prepared. io_submit return values for N_prepared == 1: + // 1 : kernel accepted; one completion will fire. // 0 or -EAGAIN : transient kernel ring full; brief in-epoch yield, then unwind. // The iocb is not queued; we still own it. @@ -381,8 +392,10 @@ Status QueueFile::ScheduleOperation(FileOperationType operationType, uint8_t* bu while (true) { result = ::io_submit(ctx, 1, iocbs); if (result == 1) break; - if (result < 0 && result != -EAGAIN) return Status::IOError; - // result == 0 (ring full) or result == -EAGAIN (kernel saying "try later") + // Nothing was queued (result != 1), so retrying can never double-submit. -EINTR is not a + // documented io_submit(2) error, but treat it as transient rather than failing the read. + if (result < 0 && result != -EAGAIN && result != -EINTR) return Status::IOError; + // result == 0 (ring full), -EAGAIN (kernel saying "try later"), or -EINTR (signal) if (retries >= kSubmitYieldBudget) { // Unwind to NativeDeviceImpl::SubmitWithEpoch to wait without holding the epoch. return Status::Pending; @@ -461,6 +474,50 @@ bool UringIoHandler::TryCompleteFor(int idx) { return false; } +// Non-blocking batch drain of ONE ring (the caller's affine ring), reaping up to kCqeBatch +// completions in a single cq_lock section with dispatch moved outside the lock. This is the +// io_uring analogue of libaio's batched TryCompleteMine (io_getevents up to +// kTryCompleteBatchEvents): the inline submitter-thread completion path (Tsavorite +// CompletePending / AsyncGetFromDisk throttle-wait) reaps its own ring a batch at a time +// instead of one io_uring_peek_cqe per call, cutting per-completion cq_lock + peek overhead ~Nx. +// Mirrors QueueRunFor's phase-2 (snapshot-before-advance, dispatch-after-release) but is a single +// non-blocking pass (no wait, no drain-until-empty loop) so it stays a bounded poll. +bool UringIoHandler::TryCompleteMineBatch(int idx) { + if (idx < 0 || idx >= static_cast(rings_.size())) return false; + struct io_uring* ring = rings_[idx]; + if (ring == nullptr) return false; + SpinLock* cq_lock = cq_locks_[idx]; + + constexpr unsigned kCqeBatch = 64; + struct io_uring_cqe* cqes[kCqeBatch]; + struct DrainSlot { + int io_res; + UringIoHandler::IoCallbackContext* context; + } snapshot[kCqeBatch]; + + cq_lock->Acquire(); + unsigned n = io_uring_peek_batch_cqe(ring, cqes, kCqeBatch); + if (n == 0) { + cq_lock->Release(); + return false; + } + for (unsigned i = 0; i < n; ++i) { + snapshot[i].io_res = cqes[i]->res; + snapshot[i].context = reinterpret_cast( + io_uring_cqe_get_data(cqes[i])); + } + io_uring_cq_advance(ring, n); + cq_lock->Release(); + + // Dispatch outside the lock. null user_data marks wake-up / rewritten-after-failed-submit SQEs + // (no caller context); skip them, exactly as TryCompleteFor / QueueRunFor do. + for (unsigned i = 0; i < n; ++i) { + if (snapshot[i].context == nullptr) continue; + DispatchUringCqe(snapshot[i].io_res, snapshot[i].context); + } + return true; +} + int UringIoHandler::QueueRun(int timeout_secs) { // Compat: drain across all rings. First ring uses the full timeout; subsequent rings poll. if (rings_.empty()) return 0; @@ -486,11 +543,27 @@ int UringIoHandler::QueueRunFor(int idx, int timeout_secs) { // Phase 1: wait up to `timeout_secs` for at least one CQE; do not consume. if (timeout_secs > 0) { - struct __kernel_timespec ts; - ts.tv_sec = timeout_secs; - ts.tv_nsec = 0; - struct io_uring_cqe* wait_cqe = nullptr; - (void)io_uring_wait_cqe_timeout(ring, &wait_cqe, &ts); + if (ext_arg_supported_) { + struct __kernel_timespec ts; + ts.tv_sec = timeout_secs; + ts.tv_nsec = 0; + struct io_uring_cqe* wait_cqe = nullptr; + (void)io_uring_wait_cqe_timeout(ring, &wait_cqe, &ts); + } else { + // Without IORING_FEAT_EXT_ARG the kernel cannot carry the timeout in io_uring_enter, so + // liburing emulates it by taking an SQE, writing a timeout request with a reserved + // user_data and flushing the SQ. That is unusable here on two counts: submitters mutate + // the same SQ under sq_lock, which this path does not hold, and the reserved user_data is + // non-null so the batch dispatch below would treat it as a caller context. Poll the CQ + // instead. The wait only saves wakeups; Wake() posts a CQE, so shutdown still unblocks + // within one sleep interval. + constexpr long kPollIntervalNs = 1000000; + const long iterations = static_cast(timeout_secs) * 1000; + for (long i = 0; i < iterations && io_uring_cq_ready(ring) == 0; ++i) { + struct timespec poll_backoff { 0, kPollIntervalNs }; + (void)nanosleep(&poll_backoff, nullptr); + } + } } // Phase 2: batch-drain. The current scheme amortizes one cq_lock acquire/release across @@ -634,7 +707,7 @@ Status UringFile::ScheduleOperation(FileOperationType operationType, uint8_t* bu if (sqe != nullptr) break; sq_lock->Release(); if (retries >= kSubmitYieldBudget) { - // Unwind to NativeDeviceImpl::SubmitWithEpoch to wait without holding the epoch. + // Sustained-full SQ ring: unwind to SubmitWithEpoch to wait without holding the epoch. return Status::Pending; } ::sched_yield(); @@ -649,52 +722,125 @@ Status UringFile::ScheduleOperation(FileOperationType operationType, uint8_t* bu } io_uring_sqe_set_data(sqe, io_context.get()); - // Submit. io_uring_submit() flushes ALL SQEs pending in this ring's SQ ring (everything between - // the kernel-consumed head and our just-prepared SQE at the tail) and returns the COUNT flushed - // — not "1 for this op". So any res >= 1 means OUR SQE (the last prepared) reached the kernel; - // there may also be a stale no-op SQE in front of it (left by a prior failed-submit/unwind or by - // Wake()), which the kernel completes harmlessly (null user_data, skipped by the drainer). - // Treating res >= 1 as success is REQUIRED for correctness: a res == 2 (stale nop + our op) taken - // as failure would rewrite/free an io_context whose op is already in flight -> use-after-free on - // completion. + // Submit. NOTE: the invariant described here holds for the NON-SQPOLL path only; the SQPOLL branch + // below has different semantics (see its inline comment). io_uring_submit() flushes SQEs pending in + // this ring's SQ ring (everything between the kernel-consumed head and our just-prepared SQE at the + // tail) and returns the COUNT consumed — + // but that count is NOT an authoritative "our SQE reached the kernel" signal: io_uring_submit() + // may PARTIALLY consume (return a positive count < pending) under kernel backpressure, so res >= 1 + // can be true while OUR tail SQE is still pending. The authoritative signal is an EMPTY SQ: + // io_uring_sq_ready(ring) == 0 means every SQE up to and including ours was consumed. // - // On transient -EAGAIN/-EBUSY (CQ ring full / kernel busy) we yield a bounded in-epoch budget, - // then UNWIND (Status::Pending) exactly like the get_sqe / libaio paths so we never spin on - // submit while holding the epoch and thread-id slot. Both the unwind path and a permanent submit - // error rewrite our prepared-but-unsubmitted SQE to a no-op with null user_data (so a later - // submit cannot dispatch a completion against the io_context we are about to free); the next - // successful submit flushes that no-op harmlessly. No-ops are bounded by the SQ ring depth and - // self-heal as soon as any submit succeeds. - int res; - int submit_retries = 0; - bool unwind = false; - while (true) { - res = io_uring_submit(ring); - if (res >= 1) break; // our SQE (the last prepared) was flushed - if (res != -EAGAIN && res != -EBUSY) break; // permanent error - if (submit_retries >= kSubmitYieldBudget) { unwind = true; break; } - sq_lock->Release(); - ::sched_yield(); - ++submit_retries; - sq_lock->Acquire(); - } - if (res < 1) { - // Permanent submit error, or we are unwinding after a sustained transient. The SQE is prepared - // in the SQ ring pointing at io_context; rewrite it to a no-op (the QueueRunFor drain loop - // skips nullptr user_data without dispatching) so a later submit cannot reference the io_context - // we free here. Safe to mutate `sqe`: we still hold sq_lock and the kernel only observes it on - // the next io_uring_submit. - io_uring_prep_nop(sqe); - io_uring_sqe_set_data(sqe, nullptr); + // CRITICAL: we hold sq_lock across the ENTIRE submit-retry burst and do NOT release it while + // yielding. If we dropped the lock, a peer submitter sharing this ring could flush our + // prepared-but-unsubmitted SQE; our own retry would then observe an empty SQ / res 0, misread it + // as "nothing submitted", rewrite our SQE to a no-op, and free an io_context whose IO is already + // in flight in the kernel -> use-after-free when the drainer dispatches the completion. Holding + // sq_lock guarantees no peer touches our SQE, so sq_ready == 0 is an unambiguous success. The + // completion drainers use a SEPARATE cq_lock (QueueRunFor / TryCompleteMineBatch), so holding + // sq_lock here never blocks CQ draining; a transient CQ-full -EAGAIN/-EBUSY clears as the drainers + // free CQ space while we yield. + // + // On sustained transient (-EAGAIN/-EBUSY past the yield budget) we UNWIND (Status::Pending) exactly + // like the get_sqe / libaio paths so we never spin on submit while holding the epoch and thread-id + // slot. Both the unwind path and a permanent submit error rewrite our still-pending SQE to a no-op + // with null user_data (skipped by the drainer) so a later submit cannot dispatch a completion + // against the io_context we are about to free; the next successful submit flushes that no-op + // harmlessly. This rewrite is safe precisely because sq_ready > 0 proves our SQE was never consumed. + int res = 0; + bool submitted; + bool permanent = false; + if (handler_->sqpoll()) { + // SQPOLL: a kernel thread polls the SQ and consumes SQEs ASYNCHRONOUSLY, so the two assumptions + // the non-SQPOLL path relies on both break: + // (1) io_uring_sq_ready() is not a synchronous "consumed" signal — it can read > 0 right after + // submit simply because the poll thread has not advanced the kernel head yet; and + // (2) the SQE must NEVER be rewritten after submit — the poll thread may read our readv/writev + // the instant io_uring_submit()'s flush advances the SQ tail, so the no-op rewrite trick + // would race the kernel and, if we then freed io_context, cause a use-after-free. + // io_uring_submit()'s internal flush UNCONDITIONALLY publishes our SQE (advances ktail to the + // tail) before any syscall; the syscall it may issue only (re)wakes a parked poll thread. Hence + // once we call submit our SQE is owned by the kernel regardless of the return value — we retry + // only to redeliver the wakeup, and always treat the op as submitted. SQ-full backpressure is + // already handled above by io_uring_get_sqe returning nullptr. + // + // Every negative return is retried, not just -EAGAIN/-EBUSY: the enter that carries + // IORING_ENTER_SQ_WAKEUP is only issued when the kernel has flagged the poll thread as parked, + // so a failure can leave the SQE published with the poller still asleep. With no later submit to + // redeliver the wakeup, that IO would never complete and Dispose's drain-wait would hang. + // Retrying is safe: liburing recomputes the pending count from the ring, so while our SQE is + // unconsumed the retry re-issues the wakeup. + // + // Retries are bounded to a few yields because the failures here are terminal rather than + // transient: an enter carrying only IORING_ENTER_SQ_WAKEUP never waits, so it cannot return + // -EINTR (documented only for IORING_ENTER_GETEVENTS), and the rest are permanent — + // -EOWNERDEAD once the poll thread has been killed, -EBADF / -ENXIO / -EINVAL / -EOPNOTSUPP + // otherwise. Waiting longer cannot turn such a failure into a success, and sq_lock and the + // epoch are held throughout, so a larger budget would only stall the ring and block epoch + // reclamation before reaching the same outcome. On exhaustion the op is still reported submitted, + // because the kernel owns the SQE. While the poll thread is merely parked the next submit on this + // ring redelivers the wakeup and the SQE is consumed then. Once it is gone (-EOWNERDEAD) nothing + // redelivers: that ring accepts IOs whose completions never arrive. Failing them individually would + // not restore correctness — everything already in flight on the ring is lost with it, and the kernel + // holds the only reference to those contexts (their user_data), so there is nothing to enumerate. + // NativeStorageDevice.Dispose bounds its drain so a ring in this state cannot hang teardown. + // The condition is reported once per device, with the errno and its consequence. + int submit_retries = 0; + while (true) { + res = io_uring_submit(ring); + if (res >= 0) break; // awake: pending count; parked: wakeup delivered + if (submit_retries >= kSubmitYieldBudget) break; // gave up redelivering; SQE already published + ::sched_yield(); + ++submit_retries; + } + if (res < 0 && handler_->TryClaimSqPollWakeFailureReport()) { + // pick_ring_index returns this thread's cached affine ring, i.e. the one just submitted to. + fprintf(stderr, + "Tsavorite native device: io_uring SQPOLL wakeup failed on ring %d with errno %d; the " + "kernel owns the submitted entry, so this ring may accept IOs whose completions never " + "arrive. Reported once per device.\n", + handler_->pick_ring_index(), -res); + } + submitted = true; + } else { + // Acceptance is decided by io_uring_sq_ready(), not by the submit return value. That helper is + // sqe_tail - *khead: it measures against the KERNEL head, not against ktail. io_uring_submit()'s + // internal flush only advances ktail, so the flush alone can never drive it to zero — only the + // kernel consuming SQEs can. sq_lock is held throughout, so our SQE is the last one produced, and + // the kernel consumes in order. Hence sq_ready == 0 proves the kernel took ours, and sq_ready > 0 + // proves it did not — covering a short submit that consumed only a preceding stale no-op, and an + // io_uring_enter that returned an error without consuming anything. + int submit_retries = 0; + while (true) { + res = io_uring_submit(ring); + if (io_uring_sq_ready(ring) == 0) break; // kernel consumed our SQE (and any stale nop) + // Reaching here means our SQE was NOT consumed, so retrying can never double-submit. -EINTR + // (io_uring_enter interrupted by a signal — routine in a managed process) is transient like + // -EAGAIN/-EBUSY: surfacing it as a permanent IO error would fail an otherwise healthy read. + if (res < 0 && res != -EAGAIN && res != -EBUSY && res != -EINTR) break; // permanent submit error + if (submit_retries >= kSubmitYieldBudget) break; // sustained transient; unwind (our SQE still pending) + ::sched_yield(); + ++submit_retries; + } + submitted = io_uring_sq_ready(ring) == 0; + permanent = !submitted && res < 0 && res != -EAGAIN && res != -EBUSY && res != -EINTR; + if (!submitted) { + // Our SQE was never consumed (sq_ready > 0 and we held sq_lock throughout). Rewrite it to a + // no-op (the drain loop skips null user_data) so a later submit cannot reference the io_context + // we free on return. Safe to mutate `sqe`: we still hold sq_lock and the kernel only observes it + // on the next io_uring_submit. + io_uring_prep_nop(sqe); + io_uring_sqe_set_data(sqe, nullptr); + } } sq_lock->Release(); - if (res < 1) { - // RAII frees io_context/caller_context_copy on return. Unwind -> SubmitWithEpoch retries the - // whole op outside the epoch; a permanent error surfaces to the caller. - return unwind ? Status::Pending : Status::IOError; + if (!submitted) { + // RAII frees io_context/caller_context_copy on return. Sustained transient -> SubmitWithEpoch + // retries the whole op outside the epoch (Pending); a permanent error surfaces to the caller. + return permanent ? Status::IOError : Status::Pending; } - // res >= 1: ownership transferred to the kernel. + // Our SQE reached the kernel: ownership transferred. caller_copy_guard.release(); io_context.release(); return Status::Ok; diff --git a/libs/storage/Tsavorite/cc/src/device/file_linux.h b/libs/storage/Tsavorite/cc/src/device/file_linux.h index 09b18df5345..829b228dc4f 100644 --- a/libs/storage/Tsavorite/cc/src/device/file_linux.h +++ b/libs/storage/Tsavorite/cc/src/device/file_linux.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -18,6 +19,12 @@ #ifdef FASTER_URING #include +// Kernel 5.11 signals via this feature bit that SQPOLL accepts non-registered file descriptors. +// The value is fixed ABI; define it when building against a liburing that predates the bit so the +// SQPOLL capability check below still compiles (and correctly refuses on such kernels). +#ifndef IORING_FEAT_SQPOLL_NONFIXED +#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) +#endif #endif #include "async.h" @@ -174,10 +181,13 @@ class QueueIoHandler { typedef QueueFile async_file_t; private: - /// Default per-context libaio ring depth. Used when the caller does not specify one. - /// The effective depth is sized up from the device throttle limit (see the 3-arg ctor and - /// NativeStorageDevice.ComputeNativeRingDepth) so that io_submit never sees a perpetually - /// full ring under a throttle > kMaxEvents; never sized below this floor. + /// Default per-context libaio ring depth. Used when the caller does not specify one (a + /// non-positive max_events). When the caller passes an explicit positive depth (the 3-arg + /// ctor) it is honored verbatim — including values BELOW this default — so the managed + /// per-device fs.aio-max-nr reservation accounting stays authoritative (see that ctor). + /// The effective depth is otherwise sized up from the device throttle limit (see the 3-arg ctor + /// and NativeStorageDevice.ResolveQueueDepth) so that io_submit never sees a perpetually + /// full ring under a throttle > kMaxEvents. constexpr static int kMaxEvents = 128; public: @@ -197,12 +207,27 @@ class QueueIoHandler { : init_errno_{ 0 } { Init(num_contexts < 1 ? 1 : num_contexts); } - /// As above, plus an explicit per-context ring depth. `max_events` is clamped up to the - /// kMaxEvents floor; callers pass NextPowerOf2(throttle_limit) so the kernel ring can hold - /// the full in-flight burst the device throttle permits, eliminating the io_submit - /// EAGAIN/ring-full backoff spin (which would otherwise pin epoch slots). + /// As above, plus an explicit per-context ring depth. A positive `max_events` is honored + /// VERBATIM (only a non-positive value falls back to the kMaxEvents default): callers pass + /// NextPowerOf2(throttle_limit) so the kernel ring can hold the full in-flight burst the device + /// throttle permits, eliminating the io_submit EAGAIN/ring-full backoff spin (which would + /// otherwise pin epoch slots). The managed layer (NativeStorageDevice.ResolveLibaioReservationDepth) + /// may deliberately pass a depth BELOW kMaxEvents to fit many coexisting single-ring devices + /// within the global fs.aio-max-nr budget; flooring it back up to kMaxEvents here would silently + /// reserve more events than the managed budget math accounted for (io_setup draws from the shared + /// budget), so fewer devices than promised would fit and later io_setup calls would fail with + /// EAGAIN. Honoring the explicit value keeps the managed reservation accounting authoritative. QueueIoHandler(size_t /*max_threads*/, int num_contexts, int max_events) - : max_events_{ max_events < kMaxEvents ? kMaxEvents : max_events } + : max_events_{ max_events > 0 ? max_events : kMaxEvents } + , init_errno_{ 0 } { + Init(num_contexts < 1 ? 1 : num_contexts); + } + + /// 5-arg overload accepted for cross-backend symmetry with UringIoHandler. libaio has no + /// submission-poll thread, so the io_uring SQPOLL parameters (`sqpoll`, `sq_thread_idle_ms`) + /// are silently ignored. + QueueIoHandler(size_t /*max_threads*/, int num_contexts, int max_events, bool /*sqpoll*/, int /*sq_thread_idle_ms*/) + : max_events_{ max_events > 0 ? max_events : kMaxEvents } , init_errno_{ 0 } { Init(num_contexts < 1 ? 1 : num_contexts); } @@ -252,9 +277,11 @@ class QueueIoHandler { /// against any cross-instance index reuse (which would be a memory-safety bug /// if A had more shards than B). /// - io_context_t pick_context() { + /// Index (into io_objects_) of the calling thread's affine context. Shared by pick_context() + /// (submit) and TryCompleteMine() (drain) so a thread reaps from the same context it submits to. + int pick_context_index() { if (io_objects_.size() == 1) { - return io_objects_[0]; + return 0; } thread_local const QueueIoHandler* tls_owner = nullptr; thread_local int tls_idx = -1; @@ -263,7 +290,21 @@ class QueueIoHandler { tls_idx = static_cast( submit_counter_.fetch_add(1, std::memory_order_relaxed) % io_objects_.size()); } - return io_objects_[tls_idx]; + return tls_idx; + } + + io_context_t pick_context() { + return io_objects_[pick_context_index()]; + } + + /// Drain ONLY the calling thread's affine context (the one pick_context() submits to). The inline + /// submitter-thread completion path (Tsavorite's CompletePending / AsyncGetFromDisk throttle-wait) + /// is the primary reaper at high IOPS; having each run thread poll just its own context issues one + /// io_getevents per poll instead of walking every context (Nx fewer syscalls and no cross-context + /// aio ring-lock contention). Coverage is preserved because each context has sharing submitters + /// and/or a dedicated drainer (QueueRunFor). Reaps a batch per call (kTryCompleteBatchEvents). + bool TryCompleteMine() { + return TryCompleteFor(pick_context_index()); } /// Invoked whenever a Linux AIO completes. @@ -465,9 +506,21 @@ class UringIoHandler { /// rounded up (RoundUpPow2) before use. constexpr static int kMaxEvents = 128; - /// Smallest power of two >= v (v already assumed > 0). Used to satisfy io_uring_queue_init's - /// power-of-two entries requirement when a throttle-derived depth is passed in. + /// Default IORING_SETUP_SQPOLL sq_thread_idle (milliseconds): how long the kernel submission-poll + /// thread keeps spinning after the last submission before parking. Used when SQPOLL is enabled and + /// the caller passes a non-positive idle value. A generous default keeps the poll thread hot across + /// brief gaps (e.g. benchmark warmup->run transitions) so submissions stay syscall-free; once parked, + /// liburing's io_uring_submit re-wakes it with a single IORING_ENTER_SQ_WAKEUP syscall. + constexpr static int kDefaultSqThreadIdleMs = 10000; + + /// Largest per-ring depth accepted. Matches io_uring's IORING_MAX_ENTRIES. Clamping here keeps + /// RoundUpPow2 from overflowing when a caller passes an out-of-range depth through the C ABI. + constexpr static int kMaxEventsLimit = 32768; + + /// Smallest power of two >= v, clamped to kMaxEventsLimit (v already assumed > 0). Used to satisfy + /// io_uring_queue_init's power-of-two entries requirement when a throttle-derived depth is passed in. static int RoundUpPow2(int v) { + if (v >= kMaxEventsLimit) return kMaxEventsLimit; int p = 1; while (p < v) p <<= 1; return p; @@ -491,11 +544,29 @@ class UringIoHandler { Init(num_rings < 1 ? 1 : num_rings); } - /// As above, plus an explicit per-ring SQ depth (rounded up to a power of two and floored at - /// kMaxEvents). Callers pass NextPowerOf2(throttle_limit) so the SQ ring can absorb the full - /// in-flight burst without io_uring_get_sqe returning null (which forces the backoff spin). + /// As above, plus an explicit per-ring SQ depth. A positive value is honoured VERBATIM apart from + /// the power-of-two rounding io_uring_queue_init requires (only a non-positive value falls back to + /// the kMaxEvents default), matching QueueIoHandler: a caller that deliberately asks for a shallow + /// ring to bound per-ring memory must get one. Callers that want the full in-flight burst to fit + /// without io_uring_get_sqe returning null (which forces the backoff spin) pass + /// NextPowerOf2(throttle_limit). UringIoHandler(size_t /*max_threads*/, int num_rings, int max_events) - : max_events_{ RoundUpPow2(max_events < kMaxEvents ? kMaxEvents : max_events) } + : max_events_{ RoundUpPow2(max_events > 0 ? max_events : kMaxEvents) } + , init_errno_{ 0 } { + Init(num_rings < 1 ? 1 : num_rings); + } + + /// As the 3-arg ctor, plus io_uring SQPOLL configuration. When `sqpoll` is true every ring is created + /// with IORING_SETUP_SQPOLL so a kernel thread polls the SQ and submissions need no io_uring_enter + /// syscall on the hot path (submit-side offload only; completion draining is unchanged). Each ring + /// gets its OWN kernel poll thread (no IORING_SETUP_ATTACH_WQ) so submission stays parallel across + /// rings — sharing a single poll thread across rings serialises submission and is a hard throughput + /// ceiling. `sq_thread_idle_ms` sets the poll thread's idle-before-park window + /// (<= 0 => kDefaultSqThreadIdleMs). + UringIoHandler(size_t /*max_threads*/, int num_rings, int max_events, bool sqpoll, int sq_thread_idle_ms) + : max_events_{ RoundUpPow2(max_events > 0 ? max_events : kMaxEvents) } + , sqpoll_{ sqpoll } + , sq_thread_idle_ms_{ sq_thread_idle_ms > 0 ? sq_thread_idle_ms : kDefaultSqThreadIdleMs } , init_errno_{ 0 } { Init(num_rings < 1 ? 1 : num_rings); } @@ -506,6 +577,8 @@ class UringIoHandler { , sq_locks_{ std::move(other.sq_locks_) } , cq_locks_{ std::move(other.cq_locks_) } , max_events_{ other.max_events_ } + , sqpoll_{ other.sqpoll_ } + , sq_thread_idle_ms_{ other.sq_thread_idle_ms_ } , init_errno_{ other.init_errno_ } { other.rings_.clear(); other.sq_locks_.clear(); @@ -538,6 +611,18 @@ class UringIoHandler { /// Number of io_uring shards. >= 1 once initialized. int num_contexts() const { return static_cast(rings_.size()); } + /// True iff these rings were created with IORING_SETUP_SQPOLL (opt-in submit-side offload). Under + /// SQPOLL a kernel thread consumes SQEs asynchronously, so the submit path must not treat + /// io_uring_sq_ready() as a synchronous "consumed" signal nor mutate an SQE after flushing it. + bool sqpoll() const { return sqpoll_; } + + /// Claims the right to report a permanent SQPOLL wakeup failure. Returns true for the first + /// caller only, so the diagnostic is emitted once per device rather than once per affected IO. + /// Cold path: only reached after the submit retry budget is exhausted. + bool TryClaimSqPollWakeFailureReport() { + return !sqpoll_wake_failure_reported_.exchange(true, std::memory_order_relaxed); + } + /// Pick a (ring, sq_lock) pair for the next submission via per-thread affinity. /// Each calling thread is assigned a ring on first call (round-robin against other /// callers) and continues to use that same ring for every subsequent submission. @@ -551,11 +636,9 @@ class UringIoHandler { /// cross-instance index reuse, which would otherwise be an out-of-bounds read on B's /// rings_/sq_locks_ when A had more rings than B. (Mirrors QueueIoHandler::pick_context.) /// - void pick_ring(struct io_uring*& ring_out, SpinLock*& lock_out) { + int pick_ring_index() { if (rings_.size() == 1) { - ring_out = rings_[0]; - lock_out = sq_locks_[0]; - return; + return 0; } thread_local const UringIoHandler* tls_owner = nullptr; thread_local int my_ring_idx = -1; @@ -564,8 +647,22 @@ class UringIoHandler { my_ring_idx = static_cast( submit_counter_.fetch_add(1, std::memory_order_relaxed) % rings_.size()); } - ring_out = rings_[my_ring_idx]; - lock_out = sq_locks_[my_ring_idx]; + return my_ring_idx; + } + + void pick_ring(struct io_uring*& ring_out, SpinLock*& lock_out) { + int idx = pick_ring_index(); + ring_out = rings_[idx]; + lock_out = sq_locks_[idx]; + } + + /// Drain ONLY the calling thread's affine ring (mirrors QueueIoHandler::TryCompleteMine). + /// Reaps a batch per call (up to kCqeBatch) in one cq_lock section with dispatch outside the + /// lock, so the inline submitter-thread completion path (Tsavorite CompletePending / + /// AsyncGetFromDisk throttle-wait) drains its own ring the same batch-at-a-time way the + /// dedicated drainer's QueueRunFor does, rather than one io_uring_peek_cqe per completion. + bool TryCompleteMine() { + return TryCompleteMineBatch(pick_ring_index()); } struct IoCallbackContext { @@ -594,6 +691,9 @@ class UringIoHandler { bool TryComplete(); /// Drain one completion from ring `idx`. bool TryCompleteFor(int idx); + /// Non-blocking batch drain of ring `idx` (the caller's affine ring): reaps up to kCqeBatch + /// completions in a single cq_lock section. Backs TryCompleteMine(). + bool TryCompleteMineBatch(int idx); /// Drain completions across all rings (back-compat for callers that do not know about sharding). int QueueRun(int timeout_secs); /// Drain completions on ring `idx` only. @@ -625,13 +725,35 @@ class UringIoHandler { for (int i = 0; i < num_rings; ++i) { auto raw_ring = new struct io_uring(); - int ret = io_uring_queue_init(max_events_, raw_ring, 0); + int ret; + unsigned params_features = 0; + if (sqpoll_) { + // IORING_SETUP_SQPOLL: a kernel thread polls this ring's SQ, so submissions are syscall-free + // while it is awake. sq_thread_idle is in milliseconds. Each ring gets its OWN poll thread + // (no IORING_SETUP_ATTACH_WQ) so submission stays parallel across rings; the kernel is free to + // place each poll thread. + struct io_uring_params params = {}; + params.flags = IORING_SETUP_SQPOLL; + params.sq_thread_idle = static_cast(sq_thread_idle_ms_); + ret = io_uring_queue_init_params(max_events_, raw_ring, ¶ms); + params_features = params.features; + } else { + ret = io_uring_queue_init(max_events_, raw_ring, 0); + } if (ret != 0) { init_errno_ = -ret; delete raw_ring; return; } rings.emplace_back(raw_ring); + if ((raw_ring->features & IORING_FEAT_EXT_ARG) == 0) ext_arg_supported_ = false; + if (sqpoll_ && (params_features & IORING_FEAT_SQPOLL_NONFIXED) == 0) { + // Before kernel 5.11 SQPOLL only accepts files registered with the ring; we submit + // ordinary descriptors, which would complete with EBADF on every IO. Refuse at init so + // the caller sees an actionable error instead of a silently failing data path. + init_errno_ = EOPNOTSUPP; + return; + } sq_locks.emplace_back(std::make_unique()); cq_locks.emplace_back(std::make_unique()); } @@ -656,9 +778,23 @@ class UringIoHandler { std::vector cq_locks_; /// Round-robin submit counter; only consulted when rings_.size() > 1. std::atomic submit_counter_{ 0 }; - /// Per-ring io_uring SQ depth passed to io_uring_queue_init(). Power of two, defaulted to - /// (and floored at) kMaxEvents; sized up from the device throttle limit by the 3-arg ctor. + /// Per-ring io_uring SQ depth passed to io_uring_queue_init(). Power of two; kMaxEvents when the + /// caller supplies no positive depth, otherwise the caller's value rounded up to a power of two. int max_events_ = kMaxEvents; + /// True iff IORING_SETUP_SQPOLL was requested: rings are created with a kernel SQ-poll thread so + /// submissions are syscall-free. Off by default (opt-in via the managed --device-uring-sqpoll knob). + bool sqpoll_ = false; + /// SQPOLL sq_thread_idle window in milliseconds (poll-thread spin-before-park). Only consulted when + /// sqpoll_ is true; the ctor floors a non-positive request at kDefaultSqThreadIdleMs. + int sq_thread_idle_ms_ = kDefaultSqThreadIdleMs; + /// True iff every ring reports IORING_FEAT_EXT_ARG (kernel 5.11+), i.e. io_uring_enter accepts the + /// wait timeout directly. Without it liburing emulates io_uring_wait_cqe_timeout by posting a + /// timeout SQE, which mutates the SQ (and its user_data) from the completion side; submitters hold + /// sq_lock while mutating the same SQ, so the drainer must not take that path. See QueueRunFor. + bool ext_arg_supported_ = true; + /// Set once a permanent SQPOLL wakeup failure has been reported, so the diagnostic is emitted a + /// single time per device rather than once per affected IO. + std::atomic sqpoll_wake_failure_reported_{ false }; /// If non-zero, the positive errno from a failed io_uring_queue_init() in the constructor. int init_errno_; }; diff --git a/libs/storage/Tsavorite/cc/src/device/file_windows.h b/libs/storage/Tsavorite/cc/src/device/file_windows.h index a02fb2b1c01..c85f8d43e0b 100644 --- a/libs/storage/Tsavorite/cc/src/device/file_windows.h +++ b/libs/storage/Tsavorite/cc/src/device/file_windows.h @@ -255,6 +255,12 @@ class ThreadPoolIoHandler { : threadpool_{ max_threads } { } + /// 5-arg overload accepted for cross-platform symmetry with UringIoHandler. Windows has no + /// io_uring submission-poll thread, so the SQPOLL parameters are ignored. + ThreadPoolIoHandler(size_t max_threads, int /*num_contexts*/, int /*max_events*/, bool /*sqpoll*/, int /*sq_thread_idle_ms*/) + : threadpool_{ max_threads } { + } + /// Move constructor. ThreadPoolIoHandler(ThreadPoolIoHandler&& other) : threadpool_{ std::move(other.threadpool_) } { @@ -327,6 +333,12 @@ class ThreadPoolIoHandler { return 1; // single IOCP per-device; sharding not applicable } + /// The Windows IOCP path completes on threadpool threads (IoCompletionCallback), so there is + /// no caller-affine ring to inline-drain; mirror TryComplete() and report nothing was reaped. + inline static constexpr bool TryCompleteMine() { + return false; + } + private: /// The parent threadpool. WindowsPtpThreadPool threadpool_; diff --git a/libs/storage/Tsavorite/cc/src/device/native_device.h b/libs/storage/Tsavorite/cc/src/device/native_device.h index 5db05a75b1c..d0ea4721e36 100644 --- a/libs/storage/Tsavorite/cc/src/device/native_device.h +++ b/libs/storage/Tsavorite/cc/src/device/native_device.h @@ -174,6 +174,8 @@ class INativeDevice { virtual int CreateDir(const std::string& dir, bool delete_existing) = 0; virtual bool TryComplete() = 0; + /// Drain only the calling thread's affine context/ring (see QueueIoHandler::TryCompleteMine). + virtual bool TryCompleteMine() = 0; virtual uint64_t GetFileSize(uint64_t segment) = 0; virtual void RemoveSegment(uint64_t segment) = 0; virtual int QueueRun(int timeout_secs) = 0; @@ -307,14 +309,18 @@ class NativeDeviceImpl : public INativeDevice { bool enablePrivileges = false, bool unbuffered = true, bool delete_on_close = false, - int max_events = 0) + int max_events = 0, + bool uring_sqpoll = false, + int uring_sqpoll_idle_ms = 0) : epoch_ { } // max_events is the per-context kernel submission-ring depth (libaio io_setup / // io_uring SQ entries). The managed wrapper sizes it up from the device throttle limit // (NextPowerOf2) so the ring can hold the full in-flight burst the throttle permits; // this keeps io_submit / io_uring_get_sqe off their EAGAIN/ring-full backoff spins, // which would otherwise pin epoch slots. <= 0 means "use the handler's default depth". - , handler_{ 16 /*max threads*/, num_io_contexts < 1 ? 1 : num_io_contexts, max_events } + // uring_sqpoll / uring_sqpoll_idle_ms are io_uring-only (IORING_SETUP_SQPOLL and the poll + // thread's idle window); the libaio / ThreadPool handlers accept and ignore them. + , handler_{ 16 /*max threads*/, num_io_contexts < 1 ? 1 : num_io_contexts, max_events, uring_sqpoll, uring_sqpoll_idle_ms } , default_file_options_{ unbuffered, delete_on_close } // FileSystemSegmentedFile validates segment_size internally (must be a positive power // of two) and throws std::invalid_argument otherwise. The C ABI wrapper wraps `new @@ -333,7 +339,8 @@ class NativeDeviceImpl : public INativeDevice { "Possible causes: (1) RLIMIT_AIO / fs.aio-max-nr exceeded " "(try: sudo sysctl -w fs.aio-max-nr=1048576); " "(2) io_uring disabled by kernel.io_uring_disabled or seccomp policy; " - "(3) kernel too old (libaio < 2.4 / io_uring < 5.1).", + "(3) kernel too old (libaio < 2.4 / io_uring < 5.1); " + "(4) --device-uring-sqpoll requires kernel 5.11+ (errno 95 / EOPNOTSUPP).", e, std::strerror(e)); init_status_ = FASTER::core::Status::IOError; return; @@ -533,6 +540,10 @@ class NativeDeviceImpl : public INativeDevice { return handler_.TryComplete(); } + bool TryCompleteMine() override { + return handler_.TryCompleteMine(); + } + uint64_t GetFileSize(uint64_t segment) override { // log_.size() can lazily OpenSegment(), whose bundle-expand path calls // epoch_->BumpCurrentEpoch() (it must publish the new file bundle and defer freeing the diff --git a/libs/storage/Tsavorite/cc/src/device/native_device_wrapper.cc b/libs/storage/Tsavorite/cc/src/device/native_device_wrapper.cc index df217a023ce..ce9cfdaa4e2 100644 --- a/libs/storage/Tsavorite/cc/src/device/native_device_wrapper.cc +++ b/libs/storage/Tsavorite/cc/src/device/native_device_wrapper.cc @@ -39,10 +39,11 @@ inline INativeDevice* FinalizeOrSurfaceError(DeviceT* device) { /// exceptions (bad_alloc, etc.) are converted similarly. template inline INativeDevice* TryConstructDevice(const char* file, uint64_t segment_size, bool omit_segment_id, int num_io_contexts, - bool enablePrivileges, bool unbuffered, bool delete_on_close, int max_events) { + bool enablePrivileges, bool unbuffered, bool delete_on_close, int max_events, + bool uring_sqpoll, int uring_sqpoll_idle_ms) { try { return FinalizeOrSurfaceError( - new DeviceT(std::string(file), segment_size, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events)); + new DeviceT(std::string(file), segment_size, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events, uring_sqpoll, uring_sqpoll_idle_ms)); } catch (const std::invalid_argument& e) { native_device::set_last_error("Invalid argument: %s", e.what()); return nullptr; @@ -147,22 +148,29 @@ extern "C" { /// burst the throttle permits, keeping io_submit / io_uring_get_sqe off their ring-full /// backoff spins. Values <= 0, or below the backend's floor, fall back to the default depth. /// Ignored on Windows (IOCP has no fixed submission-ring depth). - EXPORTED_SYMBOL INativeDevice* NativeDevice_CreateWithBackend(const char* file, bool enablePrivileges, bool unbuffered, bool delete_on_close, int32_t backend, uint64_t segment_size_bytes, bool omit_segment_id, int32_t num_io_contexts, int32_t max_events) { + /// + /// `uring_sqpoll` (0/1) is io_uring-only: when non-zero every ring is created with + /// IORING_SETUP_SQPOLL so a kernel thread polls the SQ and submissions need no io_uring_enter + /// syscall. Each ring gets its OWN poll thread (no IORING_SETUP_ATTACH_WQ) so submission stays + /// parallel across rings. `uring_sqpoll_idle_ms` is that poll thread's idle-before-park window + /// in milliseconds (<= 0 => native default). Both are ignored by the libaio and Windows backends. + EXPORTED_SYMBOL INativeDevice* NativeDevice_CreateWithBackend(const char* file, bool enablePrivileges, bool unbuffered, bool delete_on_close, int32_t backend, uint64_t segment_size_bytes, bool omit_segment_id, int32_t num_io_contexts, int32_t max_events, int32_t uring_sqpoll, int32_t uring_sqpoll_idle_ms) { native_device::clear_last_error(); if (file == nullptr) { native_device::set_last_error("NativeDevice_CreateWithBackend: 'file' argument is null."); return nullptr; } if (num_io_contexts < 1) num_io_contexts = 1; + const bool sqpoll = uring_sqpoll != 0; switch (backend) { case NativeDeviceBackend_Default: - return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events); + return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events, sqpoll, uring_sqpoll_idle_ms); #if !defined(_WIN32) && !defined(_WIN64) case NativeDeviceBackend_Libaio: - return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events); + return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events, sqpoll, uring_sqpoll_idle_ms); #ifdef FASTER_URING case NativeDeviceBackend_Uring: - return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events); + return TryConstructDevice(file, segment_size_bytes, omit_segment_id, num_io_contexts, enablePrivileges, unbuffered, delete_on_close, max_events, sqpoll, uring_sqpoll_idle_ms); #endif #endif default: @@ -233,6 +241,12 @@ extern "C" { return CABIGuard("NativeDevice_TryComplete", [&]() { return device->TryComplete(); }, false); } + /// Drain only the calling thread's affine context/ring (see INativeDevice::TryCompleteMine). + /// Used by the inline submitter-thread completion path to avoid walking every context. + EXPORTED_SYMBOL bool NativeDevice_TryCompleteMine(INativeDevice* device) { + return CABIGuard("NativeDevice_TryCompleteMine", [&]() { return device->TryCompleteMine(); }, false); + } + EXPORTED_SYMBOL uint64_t NativeDevice_GetFileSize(INativeDevice* device, uint64_t segment) { return CABIGuard("NativeDevice_GetFileSize", [&]() { return device->GetFileSize(segment); }, uint64_t{ 0 }); } diff --git a/libs/storage/Tsavorite/cc/src/device/thread.h b/libs/storage/Tsavorite/cc/src/device/thread.h index 55f5b636d57..2722d7d1c36 100644 --- a/libs/storage/Tsavorite/cc/src/device/thread.h +++ b/libs/storage/Tsavorite/cc/src/device/thread.h @@ -43,6 +43,17 @@ class Thread { static constexpr size_t kMaxNumThreads = 256; private: + /// Cache-line-padded atomic flag: exactly one slot per line. acquire_id()/release_id() run + /// on every device IO and CAS+store the calling thread's own id_used_ slot; without padding + /// the 1-byte atomics pack ~64 per line, so nearby thread indices (next_index_++ hands out + /// 0,1,2,... so the first N threads land in one line) false-share it and every submit/complete + /// ping-pongs the line across cores, a significant cost at multi-million IOPS. 128-byte + /// alignment also defeats the adjacent-line spatial prefetcher and mirrors the managed + /// NativeStorageDevice shard spacing. + struct alignas(128) PaddedAtomicFlag { + std::atomic flag; + }; + /// Encapsulates a thread ID, getting a free ID from the Thread class when the thread starts, and /// releasing it back to the Thread class, when the thread exits. class ThreadId { @@ -109,7 +120,7 @@ class Thread { uint32_t end = start + 2 * kMaxNumThreads; for (uint32_t id = start; id < end; ++id) { bool expected = false; - if (id_used_[id % kMaxNumThreads].compare_exchange_strong(expected, true)) { + if (id_used_[id % kMaxNumThreads].flag.compare_exchange_strong(expected, true)) { return id % kMaxNumThreads; } } @@ -126,8 +137,8 @@ class Thread { inline static void ReleaseEntry(uint32_t id) { assert(id != ThreadId::kInvalidId); - assert(id_used_[id].load()); - id_used_[id] = false; + assert(id_used_[id].flag.load()); + id_used_[id].flag = false; #ifdef COUNT_ACTIVE_THREADS int32_t result = --current_num_threads_; #endif @@ -139,8 +150,9 @@ class Thread { /// Next thread index to consider. static std::atomic next_index_; - /// Which thread IDs have already been taken. - static std::atomic id_used_[kMaxNumThreads]; + /// Which thread IDs have already been taken. Each flag is cache-line padded (PaddedAtomicFlag) + /// so the per-IO acquire/release CAS+store on a thread's own slot does not false-share. + static PaddedAtomicFlag id_used_[kMaxNumThreads]; #ifdef COUNT_ACTIVE_THREADS static std::atomic current_num_threads_; diff --git a/libs/storage/Tsavorite/cc/src/device/thread_manual.cc b/libs/storage/Tsavorite/cc/src/device/thread_manual.cc index 4be1598de93..7602d6a95e6 100644 --- a/libs/storage/Tsavorite/cc/src/device/thread_manual.cc +++ b/libs/storage/Tsavorite/cc/src/device/thread_manual.cc @@ -10,7 +10,7 @@ namespace core { std::atomic Thread::next_index_{ 0 }; /// No thread IDs have been used yet. -std::atomic Thread::id_used_[kMaxNumThreads] = {}; +Thread::PaddedAtomicFlag Thread::id_used_[kMaxNumThreads] = {}; #ifdef COUNT_ACTIVE_THREADS std::atomic Thread::current_num_threads_ { 0 }; diff --git a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/BenchWorker.cs b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/BenchWorker.cs index 94b856c3661..ac7965b96dc 100644 --- a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/BenchWorker.cs +++ b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/BenchWorker.cs @@ -114,8 +114,14 @@ public unsafe void Run() { while (!_benchmarkPool.TryDequeue(out op)) { + // Pool empty: every buffer this worker owns is in flight. The device's + // dedicated completion-drainer threads refill the pool via Callback, so + // this drains only as a liveness fallback. It is rarely hit because the + // per-thread throttle caps in-flight well below batchSize. Draining after + // every submit would keep it on the hot path, where the single-event + // TryComplete serializes all submitters on context 0's kernel ring mutex. + device.TryComplete(); Thread.Yield(); - continue; } long sectorCount = (long)(fileSize / sectorSize); long sector = threadRnd.NextInt64(0, (long)sectorCount) * sectorSize; @@ -123,12 +129,18 @@ public unsafe void Run() while (device.Throttle()) Thread.Yield(); localTotalSubmitted++; device.ReadAsync((ulong)sector, (IntPtr)dest, (uint)sectorSize, Callback, op); - device.TryComplete(); } } finally { - while (_benchmarkPool.Count < batchSize) Thread.Yield(); + // Drain until every buffer this worker owns has returned, polling completions so + // any reads submitted during the last iterations complete here rather than + // stranding the shutdown wait. + while (_benchmarkPool.Count < batchSize) + { + device.TryComplete(); + Thread.Yield(); + } // Authoritative throughput counter (successful ops only) is updated in the // callback. We also publish the per-thread submission count for diagnostics // (helps spot pathological submit/complete ratios under errors). diff --git a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Options.cs b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Options.cs index b1115535481..66618d2bcf2 100644 --- a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Options.cs +++ b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Options.cs @@ -18,18 +18,30 @@ public class Options [Option("file-name", Required = false, Default = "c:/data/test.dat", HelpText = "File name")] public string FileName { get; set; } - [Option("device-type", Required = false, Default = DeviceType.Native, HelpText = "Device type (Native, FileStream, RandomAccess, LocalMemory). For LocalMemory, --file-name and --io-backend are ignored.")] + [Option("device-type", Required = false, Default = DeviceType.Native, HelpText = "Device type (Native, FileStream, RandomAccess, LocalMemory). For LocalMemory, --file-name and --device-io-backend are ignored.")] public DeviceType DeviceType { get; set; } - [Option("throttle-limit", Required = false, Default = 0, HelpText = "Max device-level in-flight ops (0 = no throttle). Note: for Native libaio the kernel io_context is only 128 slots wide — running with --throttle-limit 0 plus high QD (threads × batch > 128) floods the ring and the kernel returns EAGAIN per request (surfaced as Status::IOError=4). The benchmark reports these as errors; throughput uses successful completions only. Set to 128 (matches both the libaio io_context capacity and the io_uring SQ depth this build uses) to avoid flood.")] + [Option("device-throttle-limit", Required = false, Default = 0, HelpText = "Aggregate max device-level in-flight ops (software backpressure; 0 = no throttle). Capped at device-io-contexts * device-queue-depth. This is a coarse bound, not an exact cap: it is enforced per submitter shard, so aggregate in-flight can exceed it. Exceeding the kernel ring capacity is not an error — a submit that finds the ring full unwinds and retries after a completion drains a slot.")] public int ThrottleLimit { get; set; } - [Option("completion-threads", Required = false, Default = 0, HelpText = "Number of background drainer threads that wait on IO completions (0 = processor count on Windows, 1 on Linux). On Linux Native, each drainer is bound 1:1 to its own kernel io_context (libaio) or io_uring ring; submitters distribute across contexts/rings via per-thread affinity. For DeviceType.LocalMemory, each drainer owns one SPSC ring fed by one submitter via per-thread routing. Throughput scales with this value up to the available submitter concurrency.")] + [Option("device-completion-threads", Required = false, Default = 0, HelpText = "Number of background drainer threads that wait on IO completions (0 = processor count on Windows, 1 on Linux). On Linux Native each drainer owns a contiguous slice of the --device-io-contexts rings: one ring per drainer blocks in the kernel, a slice of several polls the whole slice non-blocking so no ring is hidden behind a blocking wait. Submitters distribute across rings via per-thread affinity. For DeviceType.LocalMemory, each drainer owns one SPSC ring fed by one submitter via per-thread routing. Throughput scales with this value up to the available submitter concurrency.")] public int CompletionThreads { get; set; } - [Option("io-backend", Required = false, Default = "default", HelpText = "Linux Native IO backend: default, libaio, uring. Ignored on other devices/OSes. Unknown values are rejected at startup.")] + [Option("device-io-contexts", Required = false, Default = 0, HelpText = "Linux Native only: number of independent kernel io_contexts / io_uring rings, decoupled from --device-completion-threads. Submitters map to rings via per-thread affinity, so setting this >= submitter concurrency makes io_submit contention-free and spreads completions across more rings; the drainers each range-drain a contiguous slice. 0 (default) is backend-specific: io_uring uses a hardware-aware ring count (min(2*ProcessorCount, 64), floored at the drainer count), while libaio uses one ring per drainer. Clamped up to --device-completion-threads.")] + public int IoContexts { get; set; } + + [Option("device-queue-depth", Required = false, Default = 0, HelpText = "Linux Native only: per-ring kernel queue depth (io_uring SQ entries / libaio io_context nr_events) for each --device-io-contexts ring. 0 = default (4096). Cap 32768 (io_uring hard limit). For libaio, device-io-contexts * device-queue-depth is drawn from the global fs.aio-max-nr budget (warned/clamped if exceeded).")] + public int QueueDepth { get; set; } + + [Option("device-io-backend", Required = false, Default = "default", HelpText = "Linux Native IO backend: default, libaio, uring. Ignored on other devices/OSes. Unknown values are rejected at startup.")] public string IoBackend { get; set; } + [Option("device-uring-sqpoll", Required = false, Default = false, HelpText = "io_uring only: enable IORING_SETUP_SQPOLL so a kernel thread polls the submission queue and submissions are syscall-free. Each ring gets its own poll thread. Ignored for libaio / on Windows.")] + public bool DeviceUringSqPoll { get; set; } + + [Option("device-uring-sqpoll-idle-ms", Required = false, Default = 0, HelpText = "io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle). 0 = native default (10000). Only meaningful with --device-uring-sqpoll.")] + public int DeviceUringSqPollIdleMs { get; set; } + [Option("segment-size", Required = false, Default = 1L << 30, HelpText = "Segment size (bytes)")] public long SegmentSize { get; set; } diff --git a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Program.cs b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Program.cs index 0bf951b56a3..8e7e0f45cac 100644 --- a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Program.cs +++ b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/Program.cs @@ -67,6 +67,8 @@ static void PrintBenchMarkSummary(Options opts) Console.WriteLine($"Segment Size: {opts.SegmentSize}"); Console.WriteLine($"Throttle Limit: {opts.ThrottleLimit}"); Console.WriteLine($"Completion Threads: {opts.CompletionThreads}"); + Console.WriteLine($"IO Contexts: {opts.IoContexts}"); + Console.WriteLine($"Queue Depth: {opts.QueueDepth}"); Console.WriteLine($"Runtime: {opts.Runtime}"); Console.WriteLine($"BatchSize: {string.Join(",", opts.BatchSize.ToList())}"); Console.WriteLine($"NumThreads: {string.Join(",", opts.NumThreads.ToList())}"); @@ -220,13 +222,33 @@ static IDevice GetDevice(Options opts) if (capacity % segSize != 0) capacity = ((capacity + segSize - 1) / segSize) * segSize; int parallelism = opts.CompletionThreads > 0 ? opts.CompletionThreads : Environment.ProcessorCount; - Console.WriteLine($"[local-memory] capacity={capacity} segmentSize={segSize} parallelism(={CompletionThreadsLabel}={parallelism})"); + + // LocalMemoryDevice has no Throttle() override, so setting ThrottleLimit on it is inert: its + // in-flight bound is the per-submitter SPSC ring, whose producer blocks when full. Map the + // requested throttle onto that ring capacity (rounded up to a power of two, overflow-safe — + // the value is user-supplied, so a naive doubling loop would go negative above 2^30 and spin) + // so --device-throttle-limit means the same thing here as it does for the native device. + int localMemRing = 0; + if (opts.ThrottleLimit > 0) + { + const int MaxRing = 1 << 30; + if (opts.ThrottleLimit >= MaxRing) + localMemRing = MaxRing; + else + { + localMemRing = 1; + while (localMemRing < opts.ThrottleLimit) + localMemRing <<= 1; + } + } + + Console.WriteLine($"[local-memory] capacity={capacity} segmentSize={segSize} parallelism(={CompletionThreadsLabel}={parallelism}) ringCapacity={(localMemRing > 0 ? localMemRing : 1024)}"); return Devices.CreateLogDevice( logPath: null, deviceType: DeviceType.LocalMemory, capacity: capacity, numCompletionThreads: parallelism, - localMemorySegmentSize: segSize); + localMemoryDeviceOptions: new LocalMemoryDeviceOptions { SegmentSize = segSize, RingCapacity = localMemRing }); } var deviceType = opts.DeviceType; @@ -241,7 +263,11 @@ static IDevice GetDevice(Options opts) capacity: -1, numCompletionThreads: opts.CompletionThreads > 0 ? opts.CompletionThreads : 1, ioBackend: ParseBackend(opts.IoBackend), - logger: null), + logger: null, + numIoContexts: opts.IoContexts, + queueDepth: opts.QueueDepth, + uringSqPoll: opts.DeviceUringSqPoll, + uringSqPollIdleMs: opts.DeviceUringSqPollIdleMs), DeviceType.FileStream => new ManagedLocalStorageDevice(fileName, true, false, true, -1, false, false, false), DeviceType.RandomAccess => new RandomAccessLocalStorageDevice(fileName, true, true, true, -1, false, false, false), _ => throw new ArgumentOutOfRangeException() diff --git a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md index 5088fba8010..9c41e434218 100644 --- a/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md +++ b/libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md @@ -17,12 +17,12 @@ cd benchmark/Device.benchmark dotnet build -c Release -f net10.0 DB=bin/Release/net10.0/Device.benchmark.dll -# Linux NVMe, libaio +# Linux NVMe RAID-0, libaio numactl --membind=0 --cpunodebind=0 dotnet $DB \ - --file-name /mnt/nvme/devbench.dat --device-type Native --io-backend libaio \ - --file-size 17179869184 --sector-size 4096 \ - --batch-size 4096 --threads 16 --throttle-limit 512 --runtime 8 -# → Benchmark finished: ... throughput: ~750000 ops/sec + --file-name /raid/devbench.dat --device-type Native --device-io-backend libaio \ + --file-size 12801015808 --sector-size 512 \ + --batch-size 4096 --threads 32 --device-completion-threads 8 --device-throttle-limit 4096 --runtime 8 +# → Benchmark finished: ... throughput: ~8.2M ops/sec (8×NVMe RAID-0) ``` Always measure on a **Release** build. Run `dotnet $DB --help` for all flags. @@ -37,62 +37,134 @@ from RAM with no device.) ### NVMe storage-bound -Measured on a Dell P5600 NVMe (`fio` 4K randread ceiling ≈ 749K IOPS); reproduces -within ±2%. Common flags: `--file-size 17179869184 --sector-size 4096 ---segment-size 1073741824 --batch-size 4096 --throttle-limit 512 --runtime 8`. +Reference host: **8×NVMe SSD RAID-0** (Linux `md`, mounted `/raid`). The array is +IOPS-bound rather than bandwidth-bound at these sizes, so its `fio` random-read +ceiling is effectively the same at both block sizes used across this suite: +**8.24 M IOPS at 4 K** and **8.20 M IOPS at 512 B** (same job — 32 jobs × QD64, +`io_uring`, `O_DIRECT`, 8 files). The runs below issue 512 B reads, so **8.2 M** is +the like-for-like ceiling. The Device benchmark reaches it at the raw +`IDevice` level with **either** backend. The config is kept **compatible with the +KV/RESP benchmarks**: `--sector-size 512` (the array's logical block size — Garnet +reads the sector-aligned window covering a 128 B record) and `--file-size +12801015808` (12.8 GB = 100 M × 128 B). Common flags: `--segment-size 1073741824 +--batch-size 4096 --device-completion-threads 8 --device-throttle-limit 4096 --runtime 8`. ```bash +# libaio — a few kernel io_contexts suffice (default ct rings): numactl --membind=0 --cpunodebind=0 dotnet $DB \ - --file-name /mnt/nvme/devbench.dat --device-type Native --io-backend libaio \ - --completion-threads 1 --threads 16 \ - --file-size 17179869184 --sector-size 4096 --batch-size 4096 --throttle-limit 512 --runtime 8 + --file-name /raid/devbench.dat --device-type Native --device-io-backend libaio \ + --device-completion-threads 8 --threads 32 \ + --file-size 12801015808 --sector-size 512 --segment-size 1073741824 \ + --batch-size 4096 --device-throttle-limit 4096 --runtime 8 + +# uring — the smart default sizes rings to min(2×cores, 64) (>= submitters here), +# so it reaches the ceiling out of the box; only set --device-io-contexts for >64 submitters: +numactl --membind=0 --cpunodebind=0 dotnet $DB \ + --file-name /raid/devbench.dat --device-type Native --device-io-backend uring \ + --device-completion-threads 8 --threads 32 \ + --file-size 12801015808 --sector-size 512 --segment-size 1073741824 \ + --batch-size 4096 --device-throttle-limit 4096 --runtime 8 ``` -| backend | --completion-threads | --threads | ops/sec | +| backend | rings | --threads | ops/sec | |---|---|---|---| -| Native libaio | 1 | 16 | **750 K** | -| Native libaio | 1 | 32 | 755 K | -| Native uring | 1 | 16 | 342 K (single-ring SpinLock cap) | -| Native uring | 8 | 16 | **758 K** | +| Native libaio | ct=8 (8 io_contexts) | 32 | **8.23 M** | +| Native libaio | ct=8 | 64 | 7.7 M | +| Native uring | ct=8, default rings (smart → 64) | 32 | **8.45 M** | +| Native uring | ct=8, `--device-io-contexts 8` (under-provisioned) | 32 | 2.9 M (per-ring SpinLock cap) | +| Native uring | ct=8, `--device-io-contexts 32` | 32 | 8.00 M | + +Both backends hit the `fio` ceiling. **libaio needs only ~8 kernel io_contexts** +(its io_context mutex is cheap), whereas **io_uring needs one ring per submitter** to +escape the managed per-ring `SpinLock`. io_uring's ring count is **smart-defaulted** to +`min(2 × cores, 64)` (floored at the drainer count), so with 32 submitters it uses 64 +rings and reaches the ceiling **out of the box** (8.45 M); explicitly under-provisioning +rings below the submitter count (`--device-io-contexts 8`) exposes the SpinLock cap (~2.9 M). NUMA pinning is ~neutral at the raw device layer (node-0 +pin vs no pin within ±2%); it matters far more up the stack (KV/RESP). Peak is at +`--threads 32` (32 submit + 8 drain ≈ node-0's physical cores); `--threads 64` +oversubscribes and falls to ~7.5 M. ### Memory-device-bound (`LocalMemory`) `LocalMemory` is an in-RAM `IDevice`: reads are a `memcpy` served by per-thread -SPSC rings drained by `--completion-threads` worker threads. With no real device +SPSC rings drained by `--device-completion-threads` worker threads. With no real device latency, this measures the **submission/completion path ceiling** (ring routing, wakeups, callback dispatch) — the upper bound for `KV`/`resp` LocalMemory runs and a regression test for the ring code. ```bash -# Sweep; set --completion-threads == --threads (one SPSC ring per submitter). +# Sweep; set --device-completion-threads == --threads (one SPSC ring per submitter). for T in 8 16 32 40; do numactl --cpunodebind=0 --membind=0 dotnet $DB \ - --device-type LocalMemory --completion-threads $T --threads $T \ + --device-type LocalMemory --device-completion-threads $T --threads $T \ --file-size 1073741824 --segment-size 1073741824 --sector-size 512 \ - -b 1024 --throttle-limit 8192 --runtime 6 + -b 1024 --device-throttle-limit 8192 --runtime 6 done ``` -| --threads (= --completion-threads) | 8 | 16 | 32 | 40 | +| --threads (= --device-completion-threads) | 8 | 16 | 32 | 40 | |---|---|---|---|---| | MIOps/s | 34 | 57 | **78** | 74 | -Peaks near the physical core count, then falls off. Use a large `--throttle-limit` +Peaks near the physical core count, then falls off. Use a large `--device-throttle-limit` (8192) — there is no kernel ring to overflow, so back-pressure should not gate. ## Key knobs -- **`--throttle-limit`** — user-side in-flight cap (not a kernel limit). On fast - NVMe, 512 is safe (Little's Law keeps actual kernel in-flight ≈ 45, below the - 128-slot libaio/io_uring ring). `0` floods the ring → `code4` (EAGAIN) errors; - halve until errors disappear. For `LocalMemory`, use a large value (8192). -- **`--completion-threads`** — libaio: 1 (kernel mutex already efficient; sharding - is a no-op). io_uring: 4–8 sharded rings to escape the per-ring SpinLock. - LocalMemory: match `--threads`. -- **`--threads`** — 16 is the NVMe sweet spot; LocalMemory peaks near core count. -- **`numactl --membind=0 --cpunodebind=0`** — required; cross-NUMA costs 10–15%. +- **`--device-throttle-limit`** — user-side in-flight cap (not a kernel limit). On a fast + multi-drive array use **4096**; on a single NVMe **512** is enough (Little's Law + keeps actual kernel in-flight well below the ring depth). `0` disables the cap: this is + safe — a submit that finds the ring full unwinds and retries after a completion, it does + not error — but the retry churn costs throughput, so prefer sizing it. For `LocalMemory`, + use a large value (8192). +- **`--device-completion-threads`** — background drainer count. **8** is a good default for + a fast array (both backends); 1 suffices for a single NVMe. LocalMemory: match + `--threads`. +- **`--device-io-contexts`** — kernel io_contexts / io_uring rings, decoupled from drainers. + **libaio**: leave at default (= ct rings) — its io_context mutex is cheap, more + rings are a no-op. **io_uring**: the **smart default** already sizes rings to + `min(2 × cores, 64)`, enough for ≤ 64 submitters, so leave it unset in the common + case; set it explicitly (**>= submitter `--threads`**) only when submitters exceed 64 + or to pin an exact count. Rings **below** the submitter count share a per-ring + `SpinLock` and cap uring at ~a third of libaio. +- **`--threads`** — 32 is the NVMe-array sweet spot (submit + drain ≈ node-0 cores); + >32 oversubscribes and falls off. LocalMemory peaks near core count. +- **`numactl --membind=0 --cpunodebind=0`** — near-neutral at the raw device layer, + but keeps memory local; matters much more up the stack (KV/RESP cross-NUMA costs + 10–30%). - **`--file-size`** must be a multiple of `1024 × --sector-size`. +## Completion model & high-latency (cloud) devices + +The completion path is **block-on-signal**, not busy-spin. A read that misses +memory goes pending; the waiting thread suspends its epoch and parks on the +session's `readyResponses` semaphore (`SemaphoreSlim`, via `WaitPending`). A drainer that +owns a single ring parks in the kernel — `io_getevents(min_nr=1, timeout)` (libaio) or +`io_uring_wait_cqe_timeout` (uring) — and releases the semaphore when a completion lands. +A drainer that owns several rings (the common uring case, where the smart default creates +more rings than drainers) must not park on one of them or it would hide the siblings, so it +polls the whole slice non-blocking and sleeps 1 ms only after a sustained idle. Either way a +waiting reader burns no CPU during the device-latency window; this is the steady state on a +**high-latency (cloud) device** (Azure/EBS-class, ~0.5–2 ms), where throughput is +latency×concurrency-bound, as it must be. SQPOLL (a kernel-side submission poller) is +opt-in via `--device-uring-sqpoll` and off by default. + +Two poll levers exist only to reach the local-NVMe ceiling; neither is a hot +idle-spin, and both fall through to the block-on-signal path when completions are +not immediately ready: + +- **Inline affine drain** (always on): before parking, the reader does **one** + non-blocking peek of its own ring (`TryCompleteMine`). On a saturated fast array + the completion is usually already there, so the reader never parks (poll-driven → + peak IOPS). On a cloud device the peek usually misses and the thread parks on the + semaphore, so the one extra peek is a negligible cost there. +- **Submit-side backpressure**: `AsyncGetFromDisk` spins **only** while a thread's + in-flight exceeds its per-thread `--device-throttle-limit` share, and it drains + completions on each turn. A request/response reader holds ≤1 in-flight, so it + never hits this; it engages only for bulk multi-issue callers (recovery/scan). + Size `--device-throttle-limit` to the device's bandwidth-delay product (deep queues are + how you hide cloud latency) and the spin stays at the ceiling only. + ## Output ``` @@ -100,8 +172,9 @@ Benchmark finished: ok, err, submitted in s, throughp error breakdown: code= ... # only when err > 0 ``` -`code4` (`Status::IOError`) = kernel ring full (libaio `io_submit` EAGAIN / -io_uring SQ full). Fix by lowering `--throttle-limit`. +`code4` (`Status::IOError`) = a **permanent** submit or completion error (a non-retryable +`io_submit` / `io_uring_enter` return, or a failed IO). A transiently full kernel ring is +**not** an error — it unwinds and retries after a completion drains a slot. ## Related diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs index c96683670b9..546f110e82b 100644 --- a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs @@ -104,22 +104,22 @@ static IDevice CreateDevice(Options opts, string logPath) $"[localmemory] capacity={capacity / (1024L * 1024 * 1024)}GB segments={needSegments} segSize={segSize / (1024L * 1024)}MB parallelism(=device-completion-threads)={parallelism}{(opts.DeviceInlineCompletion ? " (inline completion)" : "")}"); // LocalMemoryDevice has no Throttle() override; its per-ring SPSC backpressure (the producer - // blocks when its ring is full) IS the in-flight bound. So map --device-throttle onto the ring + // blocks when its ring is full) IS the in-flight bound. So map --device-throttle-limit onto the ring // capacity (rounded up to a power of two) — this actually caps in-flight, with no contended // device-wide numPending counter. 0 = device default ring. int localMemRing = 0; - if (opts.DeviceThrottle > 0) + if (opts.DeviceThrottleLimit > 0) { - // Round up to a power of two, overflow-safe. --device-throttle is user-supplied, so a + // Round up to a power of two, overflow-safe. --device-throttle-limit is user-supplied, so a // naive `while (r < throttle) r <<= 1` would overflow to a negative r and spin forever // for values > 2^30; cap at the largest power-of-two int instead. const int MaxRing = 1 << 30; - if (opts.DeviceThrottle >= MaxRing) + if (opts.DeviceThrottleLimit >= MaxRing) localMemRing = MaxRing; else { localMemRing = 1; - while (localMemRing < opts.DeviceThrottle) + while (localMemRing < opts.DeviceThrottleLimit) localMemRing <<= 1; } } @@ -129,9 +129,8 @@ static IDevice CreateDevice(Options opts, string logPath) deviceType: DeviceType.LocalMemory, capacity: capacity, numCompletionThreads: parallelism, - localMemorySegmentSize: segSize, - localMemoryRingCapacity: localMemRing); - if (opts.DeviceThrottle > 0) dev.ThrottleLimit = opts.DeviceThrottle; // reflect intent in reporting (LocalMemory enforces it via the ring) + localMemoryDeviceOptions: new LocalMemoryDeviceOptions { SegmentSize = segSize, RingCapacity = localMemRing }); + if (opts.DeviceThrottleLimit > 0) dev.ThrottleLimit = opts.DeviceThrottleLimit; // reflect intent in reporting (LocalMemory enforces it via the ring) return dev; } @@ -141,7 +140,11 @@ static IDevice CreateDevice(Options opts, string logPath) deleteOnClose: true, disableFileBuffering: true, numCompletionThreads: numCt, - ioBackend: opts.ResolvedIoBackend); + ioBackend: opts.ResolvedIoBackend, + numIoContexts: opts.DeviceIoContexts, + queueDepth: opts.DeviceQueueDepth, + uringSqPoll: opts.DeviceUringSqPoll, + uringSqPollIdleMs: opts.DeviceUringSqPollIdleMs); } else if (devType == DeviceType.Null) { @@ -157,8 +160,8 @@ static IDevice CreateDevice(Options opts, string logPath) disableFileBuffering: true); } - if (opts.DeviceThrottle > 0) - dev.ThrottleLimit = opts.DeviceThrottle; + if (opts.DeviceThrottleLimit > 0) + dev.ThrottleLimit = opts.DeviceThrottleLimit; return dev; } } diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs index 79c312de246..54f3225c80a 100644 --- a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs @@ -108,7 +108,7 @@ void CsvAppendCommon(StringBuilder sb, string phase, int iteration, KvNumaPinnin .Append(_opts.RumdHasDeletes() ? "true" : "false").Append(",") .Append(KvSessionFunctions.kReaderCopyBytes).Append(",") .Append(_opts.ResolvedDeviceType).Append(",") - .Append(_opts.DeviceThrottle).Append(",") + .Append(_opts.DeviceThrottleLimit).Append(",") .Append(_opts.DeviceCompletionThreads).Append(",") .Append(_opts.ResolvedIoBackend).Append(",") .Append("basic").Append(",") diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs index 41ba7636083..01187ee7a4d 100644 --- a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs @@ -112,7 +112,7 @@ string BuildResultJson(PhaseResult r, KvNumaPinning pinning, int threadCount = 0 sb.Append("\"mutable_fraction\":0.9,"); sb.Append($"\"preallocate_log\":{(_opts.PreallocateLog ? "true" : "false")},"); sb.Append($"\"device\":\"{_opts.ResolvedDeviceType}\","); - sb.Append($"\"device_throttle\":{_opts.DeviceThrottle},"); + sb.Append($"\"device_throttle\":{_opts.DeviceThrottleLimit},"); sb.Append($"\"device_completion_threads\":{_opts.DeviceCompletionThreads},"); sb.Append($"\"device_io_backend\":\"{_opts.ResolvedIoBackend}\","); sb.Append("\"session_context\":\"basic\","); diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs index 12211ce7e22..1bd3966f8df 100644 --- a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs @@ -107,13 +107,14 @@ public class Options HelpText = "Device backend: native, randomaccess, filestream, null, localmemory, default.")] public string Device { get; set; } - [Option("device-throttle", Required = false, Default = 0, - HelpText = "Max in-flight IOs (device queue depth). 0 = device default (120 for Native; maps to " + - "the LocalMemory SPSC ring otherwise). NOTE: the 120 default under-drives a fast NVMe — " + - "set >=512 to saturate the device queue and reach its IOPS ceiling. Also size --num-keys " + - "so the log spans enough of the device (a small LBA span engages fewer NAND channels and " + - "caps IOPS below the device's large-span ceiling).")] - public int DeviceThrottle { get; set; } + [Option("device-throttle-limit", Required = false, Default = 0, + HelpText = "Aggregate max in-flight IOs (software backpressure). 0 = device default (4096 for Native; 120 for " + + "the managed in-box devices; maps to the LocalMemory SPSC ring otherwise). The Native 4096 default " + + "already saturates a fast NVMe queue; the managed 120 default under-drives one, so raise it (>=512) " + + "to reach the IOPS ceiling on those devices. Also size --num-keys so the log spans enough of the " + + "device (a small LBA span engages fewer NAND channels and caps IOPS below the device's large-span " + + "ceiling). Capped at io-contexts * queue-depth.")] + public int DeviceThrottleLimit { get; set; } [Option("device-io-backend", Required = false, Default = "default", HelpText = "Linux native backend: libaio, uring, default (=libaio).")] @@ -129,6 +130,38 @@ public class Options "Environment.ProcessorCount for LocalMemory).")] public int DeviceCompletionThreads { get; set; } + [Option("device-io-contexts", Required = false, Default = 0, + HelpText = "DeviceType.Native on Linux only: number of independent kernel io_contexts " + + "(libaio) / io_uring rings the device creates, decoupled from the number of " + + "drainer threads (--device-completion-threads). Submitters map to rings via " + + "per-thread affinity, so setting this >= submitter concurrency makes io_submit " + + "contention-free (no shared per-context aio ring/completion lock across unrelated " + + "submitters) and spreads completion posting across more rings; the drainers each " + + "range-drain a contiguous slice of the rings. 0 (default) is backend-specific: io_uring " + + "uses a hardware-aware ring count (min(2*ProcessorCount, 64), floored at the drainer count), " + + "while libaio uses one ring per drainer. Clamped up to --device-completion-threads.")] + public int DeviceIoContexts { get; set; } + + [Option("device-queue-depth", Required = false, Default = 0, + HelpText = "DeviceType.Native on Linux only: per-ring kernel queue depth (io_uring SQ entries / " + + "libaio io_context nr_events) for each of the --device-io-contexts rings. 0 = default " + + "(4096). Cap 32768 (io_uring hard limit). For libaio, io-contexts * queue-depth is drawn " + + "from the global fs.aio-max-nr budget (warned if exceeded; io_setup then fails if the budget is exhausted).")] + public int DeviceQueueDepth { get; set; } + + [Option("device-uring-sqpoll", Required = false, Default = false, + HelpText = "DeviceType.Native + --device-io-backend uring only: enable io_uring SQPOLL " + + "(IORING_SETUP_SQPOLL) so a kernel thread polls the submission queue and submissions " + + "are syscall-free. Each ring gets its own poll thread (no IORING_SETUP_ATTACH_WQ) so " + + "submission stays parallel across rings. Ignored for libaio. Off by default (opt-in).")] + public bool DeviceUringSqPoll { get; set; } + + [Option("device-uring-sqpoll-idle-ms", Required = false, Default = 0, + HelpText = "io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle): how long the " + + "kernel poll thread spins after the last submit before parking. 0 = native default (10s). " + + "Only meaningful with --device-uring-sqpoll.")] + public int DeviceUringSqPollIdleMs { get; set; } + [Option("device-inline-completion", Required = false, Default = false, HelpText = "DeviceType.LocalMemory only: complete IOs inline on the submitting thread (no " + "completion threads or rings; copy + callback run synchronously). Isolates the " + diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md index 79e3924d3ff..95b60c70233 100644 --- a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md @@ -8,7 +8,9 @@ false-sharing-free scoreboard, central tick timing). It sits one layer above [Device.benchmark](../Device.benchmark/README.md) (raw IDevice IOPS) and below [Resp.benchmark](../../../../../../benchmark/Resp.benchmark/README.md) -(full RESP server): **Resp ≤ KV ≤ Device ≤ fio**. +(full RESP server): each layer adds per-op work on top of the one below it. The +three benchmarks use different datasets and configurations, so their absolute +numbers are not directly comparable. ## Build & run @@ -26,9 +28,13 @@ counts (Server GC scales past ~8 threads). Run `dotnet $KV --help` for all flags ## The three scenarios Same dataset (100 M × 100 B), three setups distinguished by **where reads land**. -All NUMA-pin (`numactl --cpunodebind=0 --membind=0`; **required** — without it -throughput varies ~2×). Common tail: `-v 100 --rumd 100,0,0,0 --runsec 15 ---warmup-sec 5 -i 3` (`-i 3` = 3 iterations; use the `trimmed` mean). +All NUMA-pin (`numactl --cpunodebind=0 --membind=0`). Pinning matters most where +reads are served from RAM (scenarios 1 and 3), since remote-DRAM latency then gates +throughput directly; in the disk-bound scenario NVMe latency dominates and pinning +is within run-to-run noise. Common tail: `-v 100 --rumd 100,0,0,0 -i 3` (`-i 3` = 3 +iterations; use the `trimmed` mean). Scenarios 1 and 3 measure with `--runsec 15 +--warmup-sec 5`; the disk-bound scenario needs a longer window (`--runsec 25 +--warmup-sec 5`) to reach steady state — at 12 s it reads ~10% low. ### 1. Memory-bound — pure engine ceiling, no IO @@ -43,39 +49,68 @@ numactl --cpunodebind=0 --membind=0 dotnet $KV -t 32 -n 100000000 \ ### 2. NVMe storage-bound — reads hit real disk A small `--log-memory 16m` keeps ~0.125% of the dataset in RAM, so every read is a -4 KB random NVMe fetch through the pending-read path. `--device-throttle 512` is -required for peak IOPS (default 120 leaves the device idle). +random NVMe fetch through the pending-read path. On a fast array set +`--device-completion-threads 8` (the default is 1). The Native throttle already +defaults to 4096, sized for a fast NVMe queue; the managed devices +(`randomaccess`/`filestream`) default to 120 and need `--device-throttle-limit 512` +to spin up. Reference host: **8×NVMe RAID-0** (`/raid`, `fio` random-read ceiling +≈ **8.24 M IOPS at 4 K** / **8.20 M at 512 B** — the array is IOPS-bound, so block +size barely moves it); KV peaks at **~7.7 M** (≈ 94% of `fio`), close to the +raw-device ceiling measured in +[Device.benchmark](../Device.benchmark/README.md#nvme-storage-bound). ```bash +# libaio: numactl --cpunodebind=0 --membind=0 dotnet $KV -n 100000000 -v 100 \ - --device native --device-io-backend libaio --device-throttle 512 \ - --log-memory 16m --page-size 4m --segment-size 1g \ - --rumd 100,0,0,0 --load-threads 8 --run-threads-sweep 1,2,4,8,16,32 \ - --runsec 15 --warmup-sec 5 --data-path /mnt/nvme/kv + --device native --device-io-backend libaio --device-throttle-limit 4096 \ + --device-completion-threads 8 --log-memory 16m --page-size 4m --segment-size 1g \ + --rumd 100,0,0,0 --load-threads 8 --run-threads-sweep 8,32,64 \ + --runsec 25 --warmup-sec 5 -i 3 --data-path /raid/kv + +# uring: swap in --device-io-backend uring. No extra flag needed — the smart +# default sizes rings to min(2×cores, 64), covering these run-thread counts (see +# Device README); the uring rows below use it. ``` -Swap `--device-io-backend libaio` → `uring`, or `--device native` → `randomaccess` -(BCL async, slower) / `filestream` (slowest) to compare backends. Compare to the -device's `fio` ceiling (`--rw=randread --bs=4k --direct=1 --ioengine=libaio ---iodepth=64 --numjobs=8`). +Trimmed means of 3 iterations: + +| backend | pin | t=8 | t=32 | t=64 | +|---|---|---|---|---| +| libaio | node-0 | 2.37 M | 6.86 M | **7.71 M** | +| libaio | none | 2.39 M | 6.90 M | **7.65 M** | +| uring | node-0 | 2.30 M | **7.62 M** | 7.23 M | +| uring | none | 2.27 M | **7.72 M** | 7.37 M | + +libaio scales through **t=64** and peaks there (~7.7 M); uring peaks at **t=32** +(~7.6–7.7 M) and eases ~5% by t=64. Pinned and unpinned rows differ by at most ~2%, +inside run-to-run noise. Swap +`--device native` → `randomaccess` (BCL async, slower) / `filestream` (slowest) to +compare backends. Compare to the device's `fio` ceiling (`--rw=randread --bs=4k +--direct=1 --ioengine=libaio --iodepth=64 --numjobs=8`). > Confirm it's truly device-bound: on a big-RAM host the 12.8 GB dataset fits in the > page cache, but the device opens with `O_DIRECT`, so reads bypass it. During the run -> `iostat -x 1` should show `nvme r/s ≈ ops/sec` and `aqu-sz ≈ --device-throttle`; on a +> `iostat -x 1` should show `nvme r/s ≈ ops/sec` and `aqu-sz ≈ --device-throttle-limit`; on a > shared box use per-process `/proc//io` `read_bytes` (excludes other tenants). ### 3. Memory-device-bound — reads hit the in-RAM device Same as (2) but `--device localmemory`, a syscall-free RAM-backed `IDevice`. Reads still go through the full pending-read path (hash walk, `OperationState`, completion -dispatch) but with **zero disk latency**, isolating engine per-op CPU/GC. Sits -between (1) and (2), and below the +dispatch) but with **zero disk latency**, isolating engine per-op CPU/GC. It stays +below the [Device.benchmark LocalMemory ceiling](../Device.benchmark/README.md#memory-device-bound-localmemory) (which excludes the KV path). +The device copies the record on the completion thread, so a drainer pool smaller than +the run-thread count gates throughput: with `--device-completion-threads 8` this peaks +at ~3.4 M, below scenario 2. `--device-inline-completion` completes on the issuing +thread and removes that bottleneck (~19.6 M at t=32) — that is the configuration that +measures the engine's pending-read path rather than the drainer pool. + ```bash numactl --cpunodebind=0 --membind=0 dotnet $KV -n 100000000 -v 100 \ - --device localmemory --device-completion-threads 8 \ + --device localmemory --device-inline-completion \ --log-memory 16m --page-size 4m --segment-size 1g \ --rumd 100,0,0,0 --load-threads 8 --run-threads-sweep 1,2,4,8,16,32 \ --runsec 15 --warmup-sec 5 @@ -93,13 +128,23 @@ datasets touch few NAND dies and understate IOPS. `--device-io-backend`), `filestream` (slowest). - **`--log-memory`** — in-memory log window. Auto-sized to fit the dataset (reads stay in memory). Set small (`16m`) to force disk/device spill. Units: `512m`,`16g`. -- **`--device-throttle`** — max in-flight IOs. Default 120 leaves the device idle; - **use 512** to reach peak IOPS on fast NVMe. -- **`--device-completion-threads`** — native/localmemory drainer count (localmemory: - one SPSC ring per thread). +- **`--device-throttle-limit`** — max in-flight IOs. Native defaults to 4096 (sized for a + fast array); the managed devices (`randomaccess`/`filestream`) default to 120, which + leaves a fast device idle — raise to **512** on a single NVMe or **4096** on a + multi-drive array to reach peak IOPS. +- **`--device-completion-threads`** — native/localmemory drainer count (**8** on a + fast array; localmemory: one SPSC ring per thread). +- **`--device-io-contexts`** — kernel io_contexts / io_uring rings (native, decoupled + from drainers). Leave default for libaio. For **uring** the smart default sizes rings + to `min(2 × cores, 64)`, enough for ≤ 64 submitters, so leave it unset in the common + case; set it **>= run threads** only beyond 64 (or to pin an exact count) — rings below + the submitter count cap uring well below libaio (see + [Device README](../Device.benchmark/README.md#nvme-storage-bound)). - **`-b` / `--batch-size`** — run-phase batch depth (ops issued per chunk before an - opportunistic non-blocking drain). Default 1024. In-flight is bounded by - `--device-throttle`, not by this, so it is largely throughput-neutral. + opportunistic non-blocking drain). Default 1024. It sets the **per-thread buffer-rent + burst**: a thread rents one read buffer per op in the chunk before returning any, so + the batch size is what the buffer pool's per-thread reuse must cover. In-flight is + still bounded by `--device-throttle-limit`. - **`-n` keys / `-v` value-size / `--rumd` mix / `-t` threads / `-d` distribution.** ## Output diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs index 7203f968f86..239f47565ac 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs @@ -497,6 +497,9 @@ internal void TruncateUntilAddress(long toAddress) => _ = Task.Run(() => internal virtual bool TryComplete() => device.TryComplete(); + /// Drain only the calling thread's affine device completion context (see ). + internal virtual bool TryCompleteMine() => device.TryCompleteMine(); + /// Dispose allocator public virtual void Dispose() { @@ -2344,7 +2347,7 @@ internal void AsyncGetFromDisk(long fromLogicalAddress, int numBytes, AsyncIOCon { while (device.Throttle()) { - _ = device.TryComplete(); + _ = device.TryCompleteMine(); _ = Thread.Yield(); epoch.ProtectAndDrain(); } diff --git a/libs/storage/Tsavorite/cs/src/core/Device/DeviceOptions.cs b/libs/storage/Tsavorite/cs/src/core/Device/DeviceOptions.cs new file mode 100644 index 00000000000..5460db2bff3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Device/DeviceOptions.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + /// + /// Backend tuning options for on Linux (libaio / io_uring). + /// Ignored for other device types and platforms. Passed to + /// . + /// + public sealed class NativeDeviceOptions + { + /// Which IO backend (libaio or io_uring) to use. + public NativeStorageDevice.IoBackend IoBackend { get; set; } = NativeStorageDevice.IoBackend.Default; + + /// + /// Number of independent kernel io_contexts / io_uring rings (ring count), decoupled from the + /// completion-drain thread count. Set at or above submitter concurrency to make io_submit + /// contention-free and spread completion posting across more rings; the drainers each range-drain + /// a contiguous slice. 0 (default) is backend-specific: io_uring uses a hardware-aware ring count + /// (min(2 * ProcessorCount, 64), floored at the drainer count), while libaio uses one ring per + /// drainer. Clamped up to the drainer count. + /// + public int NumIoContexts { get; set; } = 0; + + /// + /// Per-ring kernel submission depth D (maxEvents for io_uring_queue_init / libaio io_setup). + /// Orthogonal to (ring count) and the aggregate throttle. + /// 0 (default) = the device default depth. + /// + public int QueueDepth { get; set; } = 0; + + /// + /// io_uring backend only: enable IORING_SETUP_SQPOLL so a kernel thread polls the submission + /// queue (syscall-free submits). Each ring gets its own poll thread. Ignored for libaio. + /// Off by default. + /// + public bool UringSqPoll { get; set; } = false; + + /// + /// io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle). Only used when + /// is true; 0 = native default. + /// + public int UringSqPollIdleMs { get; set; } = 0; + } + + /// + /// Tuning options for . Ignored for other device types. Passed to + /// . + /// + public sealed class LocalMemoryDeviceOptions + { + /// Segment size in bytes (must divide the device capacity). Default 1 GB. + public long SegmentSize { get; set; } = 1L << 30; + + /// + /// Per-submitter ring capacity (power of two), which is the device's in-flight bound (the + /// producer blocks when its ring is full). 0 (default) uses the built-in default. This is how an + /// in-flight throttle is applied to LocalMemory: its per-ring SPSC backpressure caps in-flight + /// with no device-wide counter. + /// + public int RingCapacity { get; set; } = 0; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs b/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs index fbccff73442..85b1aeab77f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs @@ -28,13 +28,12 @@ public static class Devices /// Whether we use IO completion port with polling /// Whether file buffering (during write) is disabled (default of true requires aligned writes) /// Open file in readOnly mode - /// For DeviceType.Native on Linux: which IO backend (libaio or io_uring) to use. Ignored otherwise. - /// Number of background IO completion drain threads. For DeviceType.Native on Linux: each drainer is bound 1:1 to its own kernel io_context (libaio) or io_uring ring, and submitters distribute across rings via per-thread affinity. For DeviceType.LocalMemory: each drainer owns one SPSC ring fed by one submitter via per-thread routing; pass 0 for inline completion (copy + callback run on the submitting thread, no rings/threads) or a negative value to default to . In both cases, raise this value when submitter concurrency exceeds the single-ring drain rate. Ignored otherwise. - /// For DeviceType.LocalMemory: segment size in bytes (must divide ). Default 1 GB. Ignored otherwise. - /// For DeviceType.LocalMemory: per-submitter ring capacity (power of two), which is the device's in-flight bound (the producer blocks when its ring is full). 0 = default. This is how an in-flight throttle is applied to LocalMemory: its per-ring SPSC backpressure caps in-flight with no device-wide counter. Ignored otherwise. + /// Number of background IO completion drain threads. For DeviceType.Native on Linux: a small pool of drainers that range-drain the device's kernel io_contexts (libaio) / io_uring rings; submitters distribute across rings via per-thread affinity. For DeviceType.LocalMemory: each drainer owns one SPSC ring fed by one submitter via per-thread routing; pass 0 for inline completion (copy + callback run on the submitting thread, no rings/threads) or a negative value to default to . In both cases, raise this value when submitter concurrency exceeds the single-ring drain rate. Ignored otherwise. /// Optional logger for device diagnostics. + /// For DeviceType.Native on Linux: libaio / io_uring backend tuning (IO backend, ring count, per-ring queue depth, SQPOLL). Null (default) uses the backend-specific defaults. Ignored otherwise. See . + /// For DeviceType.LocalMemory: segment size and per-ring capacity. Null (default) uses the defaults. Ignored otherwise. See . /// Device instance - public static IDevice CreateLogDevice(string logPath = null, DeviceType deviceType = DeviceType.Default, bool preallocateFile = false, bool deleteOnClose = false, long capacity = CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool useIoCompletionPort = false, bool disableFileBuffering = true, bool readOnly = false, NativeStorageDevice.IoBackend ioBackend = NativeStorageDevice.IoBackend.Default, int numCompletionThreads = 1, long localMemorySegmentSize = 1L << 30, int localMemoryRingCapacity = 0, ILogger logger = null) + public static IDevice CreateLogDevice(string logPath = null, DeviceType deviceType = DeviceType.Default, bool preallocateFile = false, bool deleteOnClose = false, long capacity = CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool useIoCompletionPort = false, bool disableFileBuffering = true, bool readOnly = false, int numCompletionThreads = 1, ILogger logger = null, NativeDeviceOptions nativeDeviceOptions = null, LocalMemoryDeviceOptions localMemoryDeviceOptions = null) { if (deviceType == DeviceType.Default) { @@ -48,16 +47,16 @@ public static IDevice CreateLogDevice(string logPath = null, DeviceType deviceTy return deviceType switch { - DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => new NativeStorageDevice(logPath, deleteOnClose, disableFileBuffering, capacity, numCompletionThreads: numCompletionThreads, ioBackend: ioBackend, logger: logger), + DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => new NativeStorageDevice(logPath, deleteOnClose, disableFileBuffering, capacity, numCompletionThreads: numCompletionThreads, ioBackend: nativeDeviceOptions?.IoBackend ?? NativeStorageDevice.IoBackend.Default, logger: logger, numIoContexts: nativeDeviceOptions?.NumIoContexts ?? 0, queueDepth: nativeDeviceOptions?.QueueDepth ?? 0, uringSqPoll: nativeDeviceOptions?.UringSqPoll ?? false, uringSqPollIdleMs: nativeDeviceOptions?.UringSqPollIdleMs ?? 0), DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => new LocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort, readOnly: readOnly, logger: logger), DeviceType.RandomAccess => new RandomAccessLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly, logger: logger), DeviceType.FileStream => new ManagedLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly, logger: logger), DeviceType.Null => new NullDevice(), DeviceType.LocalMemory => new LocalMemoryDevice( capacity: capacity, - segmentSize: localMemorySegmentSize, + segmentSize: localMemoryDeviceOptions?.SegmentSize ?? (1L << 30), parallelism: numCompletionThreads < 0 ? System.Environment.ProcessorCount : numCompletionThreads, - ringCapacity: localMemoryRingCapacity > 0 ? localMemoryRingCapacity : 1024, + ringCapacity: (localMemoryDeviceOptions?.RingCapacity ?? 0) > 0 ? localMemoryDeviceOptions.RingCapacity : 1024, fileName: logPath ?? "/userspace/ram/storage"), _ => throw new TsavoriteException($"Unsupported local device {deviceType}"), }; diff --git a/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs index af2c5d1f32c..0d1f56838e1 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs @@ -92,6 +92,20 @@ public interface IDevice : IDisposable /// bool TryComplete(); + /// + /// Try to complete async IO completions for only the calling thread's affine completion + /// context/ring, rather than scanning all of them. Used by the inline submitter-thread + /// completion path to avoid redundant per-context work when many threads drain concurrently. + /// Devices that do not shard completions may simply fall back to . + /// + /// Provided as a default interface method delegating to so that + /// existing external implementations continue to compile and behave + /// correctly without change; sharded devices (e.g. NativeStorageDevice) override it. + /// + /// + /// + bool TryCompleteMine() => TryComplete(); + /// /// Whether device should be throttled at this instant (i.e., caller should stop issuing new I/Os) /// diff --git a/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs index 178c967212e..4211bf89383 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs @@ -24,40 +24,152 @@ struct NativeResult public unsafe class NativeStorageDevice : StorageDeviceBase { /// - /// Hard ceiling on the effective in-flight throttle (max concurrent I/Os the device will drive), - /// and the upper bound the native submission ring is sized to. A configured - /// above this is clamped (and logged). + /// Default per-ring native submission depth D (io_uring SQ entries / libaio io_setup maxevents) + /// used when --device-queue-depth is not set. This is the per-ring kernel queue capacity, + /// one of the two physical device dimensions (ring COUNT = io-contexts, ring DEPTH = this). /// - /// This is a kernel-aware ceiling, not an arbitrary number: - /// - /// libaio: each io_context's io_setup(maxevents) draws from the system-wide - /// fs.aio-max-nr budget (default 65536, shared across every io_context in every process); one - /// device at this depth already takes ~1/16 of it, and deeper rings risk io_setup EAGAIN failures - /// elsewhere. - /// io_uring: IORING_MAX_ENTRIES caps a ring at 32768 entries. - /// - /// 4096 is already 8-32x a single NVMe's useful queue depth, so it is not a practical limit. If an exotic - /// configuration ever needs more, raise this constant deliberately and re-validate against the kernel - /// limits above — do not allow an unbounded throttle that would silently degrade into the submit-ring spin. + /// For io_uring this 4096 is a headroom CEILING, not pre-allocated work: the SQ is per-ring mmap + /// memory with no global budget, a ring never holds more than throttle / io-contexts in-flight, + /// and deeper rings than the workload uses cost only bounded pinned ring memory (~D*100B/ring), never + /// extra IO. For libaio it is NOT free: io_setup permanently reserves io-contexts * D + /// events from the global fs.aio-max-nr budget at creation whether used or not, so reserving the + /// full 4096/ring wastes budget and, with many coexisting devices (e.g. cluster nodes), exhausts a + /// stock 65536 budget (io_setup EAGAIN). Therefore, when queue-depth is left at the default, the libaio + /// io_setup reservation is sized DOWN to the throttle share, capped per-ring (), + /// rather than this ceiling. For multi-ring serving devices the full aggregate throttle is preserved + /// (io-contexts * reservation >= throttle) so there is no IOPS cost, unless the per-device + /// fs.aio-max-nr ceiling binds and reduces it; low-ring-count auxiliary devices (which do not + /// serve deep queues) have their reservation — and effective throttle — reduced. + /// An explicit --device-queue-depth is honored verbatim. /// - const int MaxThrottle = 1 << 12; + const int DefaultQueueDepth = 1 << 12; // 4096 /// - /// Size of the in-flight completion-tracking pool (the array). Deliberately larger - /// than so the pool is never the binding backpressure: with the effective - /// throttle capped at , in-flight settles around that value, and the extra - /// headroom absorbs the brief race where several engine threads clear the gate at - /// once — so those overshoot reads never hit the userspace slot spin-wait in . - /// The pool is pure managed memory (no kernel cost), so the headroom is cheap. + /// Per-ring headroom multiple applied when sizing the default libaio io_setup reservation to the + /// throttle share (): each ring is sized to + /// headroom * ceil(throttle / io-contexts) so an uneven submitter->ring distribution rarely + /// drives a ring to exactly-full (which would trigger a non-fatal io_submit EAGAIN unwind/retry). 2x + /// keeps a ring clear of exactly-full even under an uneven submitter-to-ring distribution. Combined with + /// (which bounds the per-ring depth) the per-device global-budget reservation is + /// min(2 * throttle, io-contexts * cap), so multi-ring serving devices keep full depth headroom + /// while low-ring-count devices stay small. /// - const int MaxResults = MaxThrottle * 2; + const int LibaioReservationHeadroom = 2; /// - /// Default per-context native submission-ring depth (libaio io_setup maxevents / io_uring - /// SQ entries) when the throttle limit does not call for a deeper ring. Matches the native - /// backend floor (file_linux.h QueueIoHandler/UringIoHandler kMaxEvents). + /// Ceiling on the per-ring default libaio io_setup reservation depth + /// (). A single libaio io_context (ring) is drained by one + /// completion thread and submitted through one kernel aio ring lock, so making a SINGLE ring hold the + /// full 4096-deep throttle is both inefficient (one drainer cannot keep a 4096-deep ring saturated) + /// and wasteful of the global fs.aio-max-nr + /// budget: deep in-flight should come from MORE rings (higher --device-completion-threads), not + /// one mega-deep ring. Capping the per-ring reservation here keeps low-ring-count devices — auxiliary + /// logs that do not serve deep random-read queues (AOF append, checkpoint bulk IO, per-node cluster + /// replication logs), which default to a single ring — small, so many of them coexist in a stock 65536 + /// budget (e.g. a multi-node cluster process opening ~15 such devices reserves ~15* + /// instead of 15*4096, which exhausts the budget). Multi-ring serving devices are unaffected by this cap: + /// at io-contexts >= 4 the 2x throttle share (2 * 4096 / io-contexts) is already <= this + /// cap, so their per-ring depth and full aggregate throttle are preserved. Only applies to the DEFAULT + /// reservation; an explicit --device-queue-depth is honored verbatim (bypasses this path). /// - const int DefaultNativeRingDepth = 128; + const int LibaioReservationCap = 1 << 11; // 2048 + + /// + /// Floor for the default libaio io_setup reservation depth (), + /// giving headroom when the throttle share is tiny (high io-contexts). Matches the native default ring depth. + /// The per-device AIO-budget ceiling applied afterwards overrides it, since exceeding the budget fails device + /// creation while a shallow ring only costs throughput. + /// + const int LibaioReservationFloor = 1 << 7; // 128 + + /// + /// Default for : the number of libaio Native device instances a single machine + /// is provisioned to coexist within the global fs.aio-max-nr budget. + /// + const int DefaultAioMaxDevices = 1 << 5; // 32 + + /// + /// Target number of libaio Native device instances a single process/machine is provisioned to coexist + /// within the global fs.aio-max-nr budget. The default per-device io_setup reservation + /// (io-contexts * queue-depth) is hard-capped at fs.aio-max-nr / this + /// (), keeping at least this many devices creatable + /// regardless of --device-completion-threads / --device-throttle-limit; see that method + /// for the two cases the cap cannot cover. This is a PROCESS-WIDE setting (not per-device) because + /// fs.aio-max-nr is a machine-global budget shared by + /// every device in every process; set it once at startup (e.g. from --device-aio-max-devices) + /// before any device is created. Because it is global, devices created through the raw + /// path (cluster auxiliary logs, AOF) honor it too, without plumbing + /// it through each call site. 32 keeps a stock 65536 budget at 2048 events/device (matching + /// ); a machine that raises fs.aio-max-nr proportionally raises + /// the per-device ceiling, so serving devices on a well-provisioned host are never starved. libaio only — + /// io_uring has no global budget (per-ring mmap), so this never applies to it. + /// + public static int AioMaxDevices = DefaultAioMaxDevices; + + /// + /// Hard cap on per-ring queue depth: io_uring's IORING_MAX_ENTRIES (32768). libaio additionally + /// draws io-contexts * queue-depth from the global fs.aio-max-nr budget (distro default + /// 65536), guarded separately at device creation. + /// + const int MaxQueueDepth = 1 << 15; // 32768 + + /// + /// Default aggregate in-flight read throttle T (software backpressure) used when + /// --device-throttle-limit is not set. This is the maximum number of disk-read IOs the + /// allocator keeps in flight before it applies backpressure (); its only + /// physical footprint is the pinned POH read buffers of the in-flight reads (~T * 4KB). It sizes + /// NOTHING in the kernel — that is 's job. + /// + /// 4096 saturates an 8-drive NVMe RAID-0 at the achievable peak (larger values do not raise it) + /// while keeping pinned read-buffer memory bounded (~16MB). High-connection deployments may raise + /// --device-throttle-limit up to the kernel capacity io-contexts * queue-depth for + /// additional throughput at very high connection counts. + /// + const int DefaultThrottleLimit = 1 << 12; // 4096 + + /// + /// Number of per-submitter-thread shards for in-flight tracking. Each submitter thread is assigned one + /// shard (round-robin) on its first IO, and every per-IO bookkeeping write (slot assignment, in-flight + /// increment/decrement) then lands on that shard's own cache lines. This removes the cache-line + /// ping-pong that a single global pending counter plus a shared free-slot queue create when dozens of + /// submitter and completion threads touch them on every IO — the dominant cost at high IOPS. + /// + /// A small fixed count suffices because gates each shard's TOTAL in-flight at + /// (≈ throttle ÷ active shards, capped at ), + /// so a shard's free-list occupancy never approaches no matter how many + /// submitter threads share it — a small shard count neither starves the free-list nor re-introduces + /// counter contention. Sized via + /// (: 2 × ProcessorCount capped at 32, the cap bounding + /// and the O(shards) scan); see that type for + /// the sizing rationale. + /// + /// + static readonly int NumShards = ConcurrencySharding.NumShardCount; + + /// + /// Number of completion-tracking slots per shard (power of two), i.e. the depth of each shard's + /// free-list (). Sized at 2x so a shard's + /// list never empties under the throttle: gates a shard's whole in-flight, so a + /// shard holds at most slots however many threads share it, leaving + /// ample free slots even accounting for the brief overshoot where a thread clears the throttle gate and + /// submits before a completion lands. + /// + const int SlotsPerShard = 256; + + /// + /// Hard cap on a single shard's (submitter thread's) in-flight IOs, enforced by . + /// Kept at half of so each shard's free-list retains 2x headroom over the + /// throttle gate, absorbing the brief overshoot where a thread clears the gate and submits before a + /// completion lands. Total in-flight across the device is still bounded by the global + /// — see . + /// + const int MaxPerThreadInFlight = SlotsPerShard / 2; + + /// + /// Size of the in-flight completion-tracking pool (the array): one entry per slot + /// across all shards. Pure managed memory (no kernel cost). Derived from . + /// + static readonly int MaxResults = NumShards * SlotsPerShard; /// /// Sentinel returned by an int-valued native entry point (currently NativeDevice_QueueRunFor) @@ -74,24 +186,88 @@ public unsafe class NativeStorageDevice : StorageDeviceBase /// const uint MinSectorSize = IDevice.MinDeviceSectorSize; - readonly ConcurrentQueue freeResults = new(); readonly ILogger logger; NativeResult[] results; /// - /// Number of pending reads on device + /// Per-shard signed in-flight IO count, sharded by submitter thread; each element () + /// is padded to its own cache-line pair so counters never false-share. A submit bumps it +1 via + /// and a native call via ; + /// the matching completion, submit error/abort unwind, or lease-release drops it −1 via + /// . It drives and 's drain-wait, + /// and its exact 0↔1 transitions maintain the live occupancy count. + /// Completion-slot assignment is handled separately by the per-shard free-lists (). + /// + ShardCounter[] shardInFlight; + + /// + /// One shard's in-flight IO counter, padded to a full cache-line pair (128 bytes) so adjacent shards' + /// counters never share a cache line — nor an adjacent-line hardware-prefetch pair. Replaces manual stride + /// indexing into a flat [] with a typed, self-describing element (cf. SpscRingState). + /// + [StructLayout(LayoutKind.Explicit, Size = 2 * CacheLineBytes)] + struct ShardCounter + { + const int CacheLineBytes = 64; + + /// Signed in-flight IO count for the shard; mutated via Interlocked, read via Volatile. + [FieldOffset(0)] public long InFlight; + + /// + /// Native-call leases held on the shard: the subset of that represents threads + /// executing inside native code right now, rather than IOs awaiting a completion. Maintained by + /// / and shares this counter's cache line, + /// so tracking it costs no additional miss. drains it separately from + /// because only the latter can be stuck forever on a lost completion. + /// + [FieldOffset(8)] public long Leases; + } + + /// + /// Per-shard free-list of completion-tracking slot offsets into . Each shard owns + /// the contiguous block [shard*SlotsPerShard, (shard+1)*SlotsPerShard); a submit rents a slot from + /// its shard's list and the slot is returned only when its IO completes (in or a + /// submit error path). This "return only after completion" invariant is what makes reuse safe under the + /// device's out-of-order completions — a monotonic counter-ring cannot, because a single slow IO can stay + /// in flight while newer submits wrap the ring back onto its slot and overwrite the still-pending + /// , delivering a stale/duplicate context on the late completion. Sharding the + /// list (rather than one global queue) keeps this off the contended cache lines at high IOPS. + /// + ConcurrentQueue[] shardFreeSlots; + + /// + /// Per-device, per-thread shard assignment. The value factory () hands out the + /// next shard round-robin the first time a thread touches this device. /// - int numPending = 0; + ThreadLocal shardIndex; + + /// Round-robin sequence used by to hand out shard indices. + int nextShardSeq; /// - /// Effective in-flight throttle used by , i.e. min(ThrottleLimit, MaxThrottle) - /// captured once when the native device is created (the same point the ring depth is fixed), so the hot - /// throttle-spin loop does not re-clamp on every call. Defaults to ; only consulted - /// once reads are in flight, by which point the native device — and this value — have been established. + /// Live count of shards currently carrying in-flight IO (≈ the number of concurrently-active submitter + /// threads), used by to split the global + /// into a per-thread in-flight budget so the device-wide + /// cap is preserved without a global in-flight counter. Maintained EXACTLY by + /// / , which bump it when a shard's in-flight transitions 0→1 (occupied) and + /// drop it when the shard returns 1→0 (idle) — the transition is detected atomically from the interlocked + /// counter's own return value, so it tracks true occupancy at all times and cannot ratchet up as the .NET + /// ThreadPool retires and re-injects submitter threads. Because gates each shard's + /// whole in-flight, dividing the budget by occupied-shard count (rather than by distinct threads) makes the + /// aggregate device cap equal the configured throttle regardless of how many threads share a shard. /// - int effectiveThrottleLimit = MaxThrottle; + int activeShards; - int resultOffset; + /// + /// Effective aggregate in-flight throttle T used by , captured once when the + /// native device is created (the same point ring count N and depth D are fixed): the configured + /// (or when unset), + /// capped at the kernel capacity N * D so aggregate in-flight can never exceed the rings. The + /// hot throttle-spin loop reads this field directly rather than recomputing. Defaults to + /// ; only consulted once reads are in flight, by which point the + /// native device — and this value — have been established. + /// + int effectiveThrottleLimit = DefaultThrottleLimit; /// /// Configuration captured at construction time; the underlying native device is created @@ -105,7 +281,19 @@ public unsafe class NativeStorageDevice : StorageDeviceBase readonly bool disableFileBuffering; readonly int numCompletionThreadsConfig; readonly int numIoContextsConfig; + readonly int numQueueDepthConfig; readonly IoBackend ioBackendConfig; + /// + /// io_uring SQPOLL opt-in (IORING_SETUP_SQPOLL). When true and the backend is io_uring, the + /// native rings are created with a kernel submission-poll thread so submissions are syscall-free + /// (submit-side offload only; completion draining is unchanged). Ignored for libaio. Off by default. + /// + readonly bool uringSqPollConfig; + /// + /// SQPOLL poll-thread idle window in milliseconds (sq_thread_idle). Only meaningful when + /// is true; 0 lets the native layer pick its default. + /// + readonly int uringSqPollIdleMsConfig; /// /// Runtime segment size in bytes that the native shim was asked to use. Populated by @@ -212,7 +400,10 @@ static IntPtr ImportResolver(string libraryName, Assembly assembly, DllImportSea try { - return NativeLibrary.Load(resolvedPath); + // Primary (Uring-capable) build. LoadWithLibaioShim also repairs a libaio SONAME + // mismatch transparently; a missing liburing.so.2 is the one failure it cannot repair + // and lets propagate, so we can fall back to the libaio-only build below. + return LoadWithLibaioShim(resolvedPath); } catch (DllNotFoundException ex) when (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && LibaioFallbackLibraryPath != null @@ -222,10 +413,14 @@ static IntPtr ImportResolver(string libraryName, Assembly assembly, DllImportSea // the Libaio backend (the default) keeps working. Selecting IoBackend.Uring at // construction time on the fallback binary throws TsavoriteException with an // install-liburing2 instruction; we never silently downgrade Uring to Libaio. + // The fallback ALSO goes through LoadWithLibaioShim, so a host that needs BOTH the + // libaio SONAME shim AND the fallback (e.g. Ubuntu 24.04+: libaio.so.1t64 + no + // liburing2) is repaired here instead of dead-ending — regardless of whether the + // dynamic loader reported the libaio or the liburing miss first. var fallbackPath = ResolveNativeLibraryPath(assembly, LibaioFallbackLibraryPath); try { - return NativeLibrary.Load(fallbackPath); + return LoadWithLibaioShim(fallbackPath); } catch (DllNotFoundException fallbackEx) { @@ -237,6 +432,23 @@ static IntPtr ImportResolver(string libraryName, Assembly assembly, DllImportSea ex); } } + } + + /// + /// Load a native device build, transparently repairing a libaio SONAME mismatch + /// (libaio.so.1 vs the 64-bit-time_t libaio.so.1t64) by dropping a compatibility symlink next + /// to the binary and retrying once. A libaio mismatch is the one link failure we can fix in + /// place; anything else — notably a missing liburing.so.2 — is allowed to propagate so the + /// caller can fall back to the libaio-only build. Shared by BOTH the primary and the fallback + /// load so a host that needs the libaio shim AND the fallback (no liburing2) is repaired + /// regardless of which unresolved SONAME the dynamic loader reports first. + /// + static IntPtr LoadWithLibaioShim(string resolvedPath) + { + try + { + return NativeLibrary.Load(resolvedPath); + } catch (DllNotFoundException ex) when (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && ex.Message.Contains("libaio.so.1", StringComparison.Ordinal)) { @@ -246,22 +458,18 @@ static IntPtr ImportResolver(string libraryName, Assembly assembly, DllImportSea // "libaio.so.1t64", so binaries built there carry a DT_NEEDED of libaio.so.1t64. // Other glibc distros (Azure Linux, RHEL, Fedora, ...) ship the historical // "libaio.so.1" instead. Whichever SONAME the loader could not resolve, drop a - // symlink of that name -> the libaio the host actually provides, next to - // libnative_device.so; the native library is built with RPATH=$ORIGIN so it picks - // the symlink up. + // symlink of that name -> the libaio the host actually provides, next to the native + // library; it is built with RPATH=$ORIGIN so it picks the symlink up. The primary and + // fallback binaries live in the same directory, so a single symlink serves both. var missingSoname = ex.Message.Contains("libaio.so.1t64", StringComparison.Ordinal) ? "libaio.so.1t64" : "libaio.so.1"; if (TryCreateLibaioCompatSymlink(resolvedPath, missingSoname, out var symlinkedPath)) { - try - { - return NativeLibrary.Load(resolvedPath); - } - catch (DllNotFoundException) - { - // Fall through to the detailed error below. - } + // Retry once with the symlink in place. If this build ALSO needs a library we + // cannot repair here (e.g. liburing.so.2), that DllNotFoundException propagates so + // the caller can try the libaio-only fallback build. + return NativeLibrary.Load(resolvedPath); } throw new DllNotFoundException(BuildLibaioDiagnostic(symlinkedPath, missingSoname, ex), ex); @@ -540,8 +748,10 @@ public enum IoBackend : int Uring = 2, } + // The native creator takes the full parameter list including the io_uring SQPOLL knobs; + // libaio and Windows ignore the SQPOLL arguments. [DllImport(NativeLibraryName, EntryPoint = "NativeDevice_CreateWithBackend", CallingConvention = CallingConvention.Cdecl)] - static extern IntPtr NativeDevice_CreateWithBackend(string file, bool enablePrivileges, bool unbuffered, bool delete_on_close, int backend, ulong segmentSizeBytes, bool omitSegmentIdFromFilename, int numIoContexts, int maxEvents); + static extern IntPtr NativeDevice_CreateWithBackend(string file, bool enablePrivileges, bool unbuffered, bool delete_on_close, int backend, ulong segmentSizeBytes, bool omitSegmentIdFromFilename, int numIoContexts, int maxEvents, int uringSqPoll, int uringSqPollIdleMs); [DllImport(NativeLibraryName, EntryPoint = "NativeDevice_GetSegmentSize", CallingConvention = CallingConvention.Cdecl)] static extern ulong NativeDevice_GetSegmentSize(IntPtr device); @@ -570,6 +780,9 @@ public enum IoBackend : int [DllImport(NativeLibraryName, EntryPoint = "NativeDevice_TryComplete", CallingConvention = CallingConvention.Cdecl)] static extern bool NativeDevice_TryComplete(IntPtr device); + [DllImport(NativeLibraryName, EntryPoint = "NativeDevice_TryCompleteMine", CallingConvention = CallingConvention.Cdecl)] + static extern bool NativeDevice_TryCompleteMine(IntPtr device); + [DllImport(NativeLibraryName, EntryPoint = "NativeDevice_QueueRun", CallingConvention = CallingConvention.Cdecl)] static extern int NativeDevice_QueueRun(IntPtr device, int timeout_secs); @@ -631,18 +844,197 @@ static string FormatNativeError() readonly AsyncIOCallback _callbackDelegate; CancellationTokenSource completionThreadToken; Thread[] completionThreads; + int numRingsActual; + + /// Shard index for the calling thread on this device (assigned on first access). + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + int GetShard() => shardIndex.Value; + + /// + /// value factory: hands out the next shard round-robin. The round-robin + /// counter is reduced modulo as uint so that when the signed + /// eventually wraps past on a long-lived, + /// thread-churning server the index stays in [0, NumShards) (a signed % would yield a + /// negative index and an out-of-range shard access). need not be a power of two. + /// + int AssignShard() + => (int)((uint)(Interlocked.Increment(ref nextShardSeq) - 1) % (uint)NumShards); + + /// + /// Records a submit (or lease) on : bumps the shard's in-flight counter and, when + /// it transitions 0→1 (the shard becomes occupied), bumps . The 0→1 test reads + /// the interlocked increment's own return value, so exactly one caller observes the transition even under + /// concurrent submit/complete on the same shard — keeping an exact, always-current + /// live-occupancy count (no background reconciliation, no birth-only ratchet under ThreadPool churn). + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void SubmitToShard(int shard) + { + if (Interlocked.Increment(ref shardInFlight[shard].InFlight) == 1L) + Interlocked.Increment(ref activeShards); + } - // Instrumentation: peak concurrent in-flight writes seen, and submit/complete counters. - // Set TSAVORITE_DEVICE_INSTRUMENT=1 in the environment to enable. - static readonly bool s_instrument = Environment.GetEnvironmentVariable("TSAVORITE_DEVICE_INSTRUMENT") == "1"; - int peakNumPending; - long submitCount; - long completeCount; - long submitNanos; + /// + /// Records a completion (or lease-release / submit-error unwind) on : drops the + /// shard's in-flight counter and, when it returns to 0 (the shard becomes idle), drops + /// . Balances ; see it for why the transition is exact. + /// Every completion is preceded by its submit, so a shard's in-flight is never negative and + /// is never over-decremented. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void CompleteShard(int shard) + { + if (Interlocked.Decrement(ref shardInFlight[shard].InFlight) == 0L) + Interlocked.Decrement(ref activeShards); + } + + /// + /// Rents a free completion-tracking slot offset from the calling submitter's shard. Under the throttle a + /// shard never holds more than slots at once (< ), + /// so the fast path always succeeds; the spin is a safety net for + /// the rare unthrottled bulk caller (e.g. page reads on recovery). Completions run on separate drainer + /// threads that return slots via , so the spin cannot self-deadlock. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + int RentSlot(int shard) + { + var freeList = shardFreeSlots[shard]; + if (freeList.TryDequeue(out int offset)) + return offset; + var spin = new SpinWait(); + while (!freeList.TryDequeue(out offset)) + spin.SpinOnce(); + return offset; + } + + /// + /// Returns a completion-tracking slot to its owning shard's free-list. The owning shard is encoded in the + /// offset (offset / SlotsPerShard), so a completion on any drainer thread returns the slot to the + /// list the submitter will rent from — no cross-shard mixing. Called only after the slot's IO has + /// completed (its has been read), preserving the "reuse only after completion" + /// invariant that makes slot reuse safe under out-of-order completions. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void ReturnSlot(int offset) + { + // Release the completed slot's captured callback delegate and context object before the slot + // is re-enqueued, so neither is kept rooted until the slot is next rented (otherwise up to + // MaxResults user contexts stay reachable indefinitely on a mostly-idle device). Every caller + // has already consumed the NativeResult first (the `var result = results[offset]` copy in + // _callback, and the local callback/context parameters on the error/disposed paths), and clearing + // BEFORE the enqueue means a concurrent RentSlot that dequeues this offset and writes its own + // NativeResult cannot be clobbered by this clear. + results[offset] = default; + shardFreeSlots[offset / SlotsPerShard].Enqueue(offset); + } + + /// Current in-flight IO count for a shard. + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + long InFlightInShard(int shard) + => Volatile.Read(ref shardInFlight[shard].InFlight); + + /// Sum of in-flight IOs across all shards. Cold path only (Dispose drain, instrumentation). + long TotalInFlight() + { + long total = 0; + for (int s = 0; s < NumShards; s++) + total += Volatile.Read(ref shardInFlight[s].InFlight); + return total; + } + + /// + /// Marks the calling thread as executing inside a native call on . Bumps the shard's + /// in-flight counter — so and observe the lease exactly as + /// a submit does — and, on the same cache line, the lease counter drains separately. + /// Bumping in-flight first keeps leases ≤ in-flight at every instant, so an in-flight count of zero proves + /// no native call is in progress. Caller MUST balance this with in a finally. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void EnterNativeCall(int shard) + { + SubmitToShard(shard); + Interlocked.Increment(ref shardInFlight[shard].Leases); + } + + /// + /// Balances . Drops the lease before the in-flight count, preserving the + /// leases ≤ in-flight invariant. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void ExitNativeCall(int shard) + { + Interlocked.Decrement(ref shardInFlight[shard].Leases); + CompleteShard(shard); + } + + /// Sum of native-call leases across all shards. Cold path only (Dispose teardown). + long TotalLeases() + { + long total = 0; + for (int s = 0; s < NumShards; s++) + total += Volatile.Read(ref shardInFlight[s].Leases); + return total; + } + + /// + /// Per-thread in-flight budget = global throttle split across the currently-occupied shards + /// (≈ live concurrent submitters, tracked exactly by ), clamped to + /// . Each shard admits against this budget independently, so the + /// device-wide in-flight count tracks the configured + /// only approximately: a shard admits against the divisor in effect at the time, so shards that + /// filled while few were occupied keep the larger budget they were granted. Aggregate in-flight is + /// therefore bounded absolutely by × , + /// and a low configured throttle can be exceeded by several times if shards become occupied one at + /// a time without completions in between. This is intentional — the throttle is a coarse + /// backpressure/memory bound, and exact kernel-queue-capacity safety is enforced downstream by + /// the native ring-full retry (a submit that finds the SQ/io_context ring full unwinds to Pending and waits + /// for a completion before retrying), not by the precision of this split. Keeping the check per-shard lets + /// the hot path read only the calling thread's own cache lines. + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + int PerThreadLimit() + { + int global = Volatile.Read(ref nativeDevice) == IntPtr.Zero + ? (ThrottleLimit > 0 ? ThrottleLimit : DefaultThrottleLimit) + : effectiveThrottleLimit; + int active = Volatile.Read(ref activeShards); + if (active < 1) active = 1; + int perThread = global / active; + if (perThread < 1) perThread = 1; + if (perThread > MaxPerThreadInFlight) perThread = MaxPerThreadInFlight; + return perThread; + } + + /// + /// Leases the native handle for a non-IO native call (TryComplete / Reset / RemoveSegment / GetFileSize) + /// so a concurrent cannot free it mid-call. Returns false (without leasing) once + /// disposal has begun. On success the caller MUST call in a finally. The lease + /// bumps the shard in-flight counter, so Dispose's drain-wait covers leased native calls automatically, + /// and the shard lease counter, which gates handle destruction even if that drain hits its deadline. + /// + bool TryLease(out int shard) + { + shard = GetShard(); + if (Volatile.Read(ref disposedFlag) != 0) + return false; + EnterNativeCall(shard); + // Re-check after publishing the lease: if Dispose set the flag concurrently, its drain-wait either + // already observed this lease (and is waiting) or will not — either way we must not touch the handle. + if (Volatile.Read(ref disposedFlag) != 0) + { + ExitNativeCall(shard); + return false; + } + return true; + } + + /// Releases a lease taken by . + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + void ReleaseLease(int shard) + => ExitNativeCall(shard); void _callback(IntPtr context, int errorCode, ulong numBytes) { - if (s_instrument) Interlocked.Increment(ref completeCount); int offset = (int)context; var result = results[offset]; // CRITICAL: this method is invoked via a function pointer from native code (libaio / @@ -651,7 +1043,7 @@ void _callback(IntPtr context, int errorCode, ulong numBytes) // loop and, when it crosses the ABI boundary, causes the .NET runtime to // terminate the drainer thread (silently, since it's a background thread). That // leaves the device with no completion processor: all subsequent IOs are - // submitted but never completed, numPending grows unbounded, device.Throttle() + // submitted but never completed, in-flight grows unbounded, device.Throttle() // stays true forever, and the next worker thread to call ReadAsync deadlocks // spinning in the throttle-wait loop. // @@ -662,98 +1054,174 @@ void _callback(IntPtr context, int errorCode, ulong numBytes) // Errors that need surfacing should go through the result.callback's own error // channel (numBytes=0 + errorCode), not via a throw. // - // try/finally also ensures that on a throwing user callback the result slot is - // returned AND numPending is decremented. Dispose() spins until numPending == 0, - // so decrementing here (after the callback returns) guarantees Dispose waits for - // all in-flight user callbacks to finish before destroying the native device - // underneath them. + // try/finally also ensures that on a throwing user callback the shard's in-flight + // counter is still dropped. Dispose() spins until in-flight reaches 0, so dropping + // here (after the callback returns) guarantees Dispose waits for all in-flight user + // callbacks to finish before destroying the native device underneath them. try { result.callback((uint)errorCode, (uint)numBytes, result.context, ioException: default); } catch (Exception ex) { - logger?.LogCritical(ex, "Unhandled exception in user IO completion callback (suppressed to keep drainer alive)"); + // The logger is host-supplied and can itself throw (for example a provider whose sink was + // disposed during shutdown). An exception escaping this handler would defeat the firewall + // described above, so the report is guarded too. + try + { + logger?.LogCritical(ex, "Unhandled exception in user IO completion callback (suppressed to keep drainer alive)"); + } + catch + { + // Nothing further can be reported from inside a completion callback. + } } finally { - freeResults.Enqueue(offset); - Interlocked.Decrement(ref numPending); + // The owning shard is encoded in the slot offset (offset / SlotsPerShard), not GetShard(): this + // completion may run on a drainer thread whose own shard differs from the submitter's. + CompleteShard(offset / SlotsPerShard); + ReturnSlot(offset); } } - /// Diagnostic: snapshot and reset per-second submit/complete counters and peak in-flight. - /// Set environment variable TSAVORITE_DEVICE_INSTRUMENT=1 to enable population. - public (int curPending, int peakPending, long submits, long completes, long submitNs) GetAndResetStats() - { - var stats = (numPending, peakNumPending, submitCount, completeCount, submitNanos); - peakNumPending = numPending; - submitCount = 0; - completeCount = 0; - submitNanos = 0; - return stats; - } - /// /// - /// Gates on , the configured - /// clamped to and captured once at device creation. A configured throttle above - /// that ceiling cannot be honored (the kernel submission ring is capped there — see ), - /// so gating on the raw value would let the engine drive more in-flight reads than the ring can hold and push - /// them into the submit-ring spin. The clamp is precomputed (not recomputed per call) because this runs in the - /// engine's throttle-spin loop. + /// In-flight is tracked per submitter thread (shard) to avoid the cache-line contention a single global + /// counter creates at high IOPS. Each thread throttles on its own shard's in-flight count against a + /// per-thread budget () derived from the effective aggregate throttle + /// (, or when unset, + /// capped at the kernel capacity io-contexts * queue-depth) split across the active submitter threads — + /// so the device-wide in-flight count closely tracks the effective throttle (within ≈ one IO per occupied + /// shard at the boundary; see ), while this hot check only reads the calling + /// thread's own cache lines. The throttle is coarse admission backpressure, not a hard kernel-capacity gate: + /// the native ring-full retry (submit unwinds to Pending and waits for a completion when the ring is full) + /// is what guarantees the SQ/io_context ring never overflows regardless of this split's precision. /// - /// Before the native device is lazily created (cold start), still holds - /// its seed, which would let a startup burst of concurrent submitters bypass the - /// configured throttle and flood the just-sized ring. So until the handle exists we gate on the live clamped - /// limit; the predictable branch is paid only on the cold path. + /// Before the native device is lazily created (cold start), gates on the live + /// clamped limit rather than the seeded , so a startup burst of + /// concurrent submitters cannot bypass the configured throttle and flood the just-sized ring. /// /// public override bool Throttle() - => numPending > (Volatile.Read(ref nativeDevice) == IntPtr.Zero - ? Math.Min(ThrottleLimit, MaxThrottle) - : effectiveThrottleLimit); - - /// - /// Computes the per-context native kernel submission-ring depth from . - /// The rings (one per io_context) must, in aggregate, hold the in-flight burst permits: - /// if a ring is smaller than the load it sees, io_submit / io_uring_get_sqe spin in their ring-full backoff while - /// holding a native epoch slot, and under enough concurrent submitters that starves the native epoch-slot table - /// (the libaio hang this addresses). Submitters spread across the contexts by thread affinity, so each context - /// handles ~ThrottleLimit / numIoContexts in steady state; the per-context depth is sized to that share - /// (rounded up to a power of two) rather than the full throttle, so the total kernel capacity - /// (numIoContexts × depth) stays bounded — libaio's io_setup draws from the shared fs.aio-max-nr - /// budget, so applying the full throttle to every context could exhaust it with many completion threads. The - /// floor adds headroom against uneven distribution (a brief ring-full is a - /// non-fatal unwind-and-retry, not a hang); the cap is the kernel-safe maximum (see ). - /// For the default single context this is max(DefaultNativeRingDepth, NextPowerOf2(ThrottleLimit)), clamped to - /// . Read at native-device creation time (first IO), by which point the factory has applied - /// the configured throttle. Ignored by the Windows (IOCP) backend. - /// - int ComputeNativeRingDepth() { - int throttle = ThrottleLimit; - if (throttle <= 0) - return DefaultNativeRingDepth; - if (throttle > MaxThrottle) + int shard = GetShard(); + return InFlightInShard(shard) > PerThreadLimit(); + } + + /// + /// Resolves the per-ring native submission depth D (maxEvents passed to io_uring_queue_init / + /// libaio io_setup). Returns --device-queue-depth when set (clamped to + /// ), else . This is the ring DEPTH knob — + /// orthogonal to the ring COUNT (io-contexts) and the aggregate in-flight throttle. Read at + /// native-device creation (first IO). Ignored by the Windows (IOCP) backend. + /// + int ResolveQueueDepth() + { + int d = numQueueDepthConfig > 0 ? numQueueDepthConfig : DefaultQueueDepth; + if (d > MaxQueueDepth) + { + logger?.LogWarning( + "NativeStorageDevice: queue-depth ({depth}) exceeds the io_uring maximum ({max}); clamping.", + d, MaxQueueDepth); + d = MaxQueueDepth; + } + return d; + } + + /// + /// Per-ring libaio io_setup reservation depth when --device-queue-depth is left at the + /// default. Unlike io_uring (per-ring mmap, no global budget), libaio permanently reserves + /// io-contexts * depth events from the shared fs.aio-max-nr budget at creation, so the + /// ceiling (4096) over-reserves: a libaio ring never holds more than the + /// aggregate throttle spread across the rings. We size the reservation to that share — + /// NextPow2(headroom * ceil(throttle / io-contexts)), floored at , + /// capped at (a single ring should not hold the full deep queue — see + /// that constant) and at the (the resolved queue-depth) — so the per-device + /// reservation is io-contexts * result = min(headroom * throttle, io-contexts * cap). Multi-ring + /// serving devices (io-contexts >= 4) keep io-contexts * result >= throttle by the share math (the + /// throttle share is <= the cap), so the full aggregate throttle stays usable (effectiveThrottleLimit is + /// NOT reduced => no IOPS cost); low-ring-count auxiliary devices drop to ~= io-contexts * cap, + /// letting many coexist in a stock 65536 budget. Finally the WHOLE-device reservation is hard-capped at + /// fs.aio-max-nr / AioMaxDevices (default ) so at least that many + /// devices fit the kernel budget; on a stock 65536 budget this bounds each device to 2048 events, while a + /// host that sizes fs.aio-max-nr for its workload keeps serving devices at full depth (e.g. 4194304 / 32 = + /// 131072 per device, which never binds). That ceiling runs last and overrides the share math above: when + /// it binds, effectiveThrottleLimit drops with it (a stock 65536 budget halves the default 4096 throttle at + /// every ring count), so size fs.aio-max-nr such that fs.aio-max-nr / AioMaxDevices >= throttle. The cap + /// is best-effort in two further respects: depth cannot fall below one event per ring, so a device + /// configured with more rings than its per-device share still exceeds it + /// (warned); and the budget is the machine total, not what remains after other processes' reservations. + /// + int ResolveLibaioReservationDepth(int ringCount, int throttle, int ceilingDepth) + { + long share = ((long)throttle + ringCount - 1) / ringCount; // ceil(throttle / ringCount) + long depth = Utility.NextPowerOf2(share * LibaioReservationHeadroom); + if (depth < LibaioReservationFloor) depth = LibaioReservationFloor; + if (depth > LibaioReservationCap) depth = LibaioReservationCap; + if (depth > ceilingDepth) depth = ceilingDepth; + + // Hard per-device AIO budget: keep at least AioMaxDevices libaio devices fitting the global + // fs.aio-max-nr budget by bounding this device's WHOLE reservation (ringCount * depth), independent + // of throttle. Halve the depth (staying a power of two) until it fits; this wins over the soft floor + // above. The caller then caps effectiveThrottleLimit at ringCount * depth, so aggregate in-flight + // tracks the (possibly reduced) reservation. + int maxDevices = AioMaxDevices < 1 ? DefaultAioMaxDevices : AioMaxDevices; + long perDeviceBudget = GetAioMaxNr() / maxDevices; + while (depth > 1 && (long)ringCount * depth > perDeviceBudget) + depth >>= 1; + + // One event per ring is the floor, so a device with more rings than its per-device share cannot be + // brought within the bound by depth alone. Surface it: the operator must lower --device-io-contexts + // or raise fs.aio-max-nr / --device-aio-max-devices, or io_setup may fail with EAGAIN. + if ((long)ringCount * depth > perDeviceBudget) { - // Don't silently ignore the configured throttle: surface that it exceeds the device's - // maximum supported in-flight depth and is being clamped. The ceiling is the kernel-safe - // submission-ring depth (io_uring max SQ entries / shared libaio fs.aio-max-nr budget), and - // already exceeds practical NVMe queue depths, so raising it is rarely useful. logger?.LogWarning( - "NativeStorageDevice: ThrottleLimit ({throttle}) exceeds the device's maximum in-flight I/O depth ({max}); effective in-flight is capped at that maximum.", - throttle, MaxThrottle); - throttle = MaxThrottle; + "NativeStorageDevice: libaio reservation ({rings} rings x {depth} events = {total}) exceeds the " + + "per-device share of fs.aio-max-nr ({budget} = {aioMaxNr} / {maxDevices}); lower --device-io-contexts, " + + "raise fs.aio-max-nr, or lower --device-aio-max-devices.", + ringCount, depth, (long)ringCount * depth, perDeviceBudget, GetAioMaxNr(), maxDevices); + } + + return (int)depth; + } + + /// + /// Best-effort read of the global libaio event budget fs.aio-max-nr (distro default 65536, shared + /// across every process on the machine). Returns the stock 65536 default if the /proc entry is unreadable + /// (e.g. non-Linux). Never throws. Read at device creation (rare), so not cached. + /// + static long GetAioMaxNr() + { + try + { + return long.Parse(System.IO.File.ReadAllText("/proc/sys/fs/aio-max-nr").Trim()); + } + catch + { + return 1 << 16; // stock default fallback (best-effort) + } + } + + /// + /// Best-effort libaio guard: io_setup draws io-contexts * queue-depth events from the + /// global fs.aio-max-nr budget (distro default 65536, shared across every process). If the + /// requested total exceeds that budget, warn up front so the operator sees an actionable message + /// rather than a cryptic io_setup EAGAIN at device creation. io_uring uses per-ring mmap memory + /// only (no global budget), so this applies to libaio only. Never throws (best-effort /proc read). + /// + void WarnIfLibaioAioBudgetExceeded(int numContexts, int queueDepth) + { + if (ioBackendConfig != IoBackend.Libaio && ioBackendConfig != IoBackend.Default) + return; + long requested = (long)numContexts * queueDepth; + long budget = GetAioMaxNr(); + if (requested > budget) + { + logger?.LogWarning( + "NativeStorageDevice: libaio io-contexts*queue-depth ({req} = {n}*{d}) exceeds the system fs.aio-max-nr budget ({budget}); io_setup may fail. Lower --device-io-contexts or --device-queue-depth, or raise fs.aio-max-nr.", + requested, numContexts, queueDepth, budget); } - int contexts = numIoContextsConfig < 1 ? 1 : numIoContextsConfig; - int perContext = (throttle + contexts - 1) / contexts; // ceil(throttle / contexts) - int depth = (int)Utility.NextPowerOf2(perContext); - if (depth < DefaultNativeRingDepth) - depth = DefaultNativeRingDepth; - if (depth > MaxThrottle) - depth = MaxThrottle; - return depth; } /// @@ -791,13 +1259,21 @@ public static (bool defaultAvailable, bool uringAvailable) GetAvailableBackends( /// treated as 1. /// IO backend to use (default platform backend, or explicit libaio / io_uring on Linux). /// + /// Number of independent kernel io_contexts (libaio) / io_uring rings, decoupled from the drainer count. 0 = device default. + /// Per-ring kernel submission depth (maxEvents). 0 = device default. + /// io_uring only: enable IORING_SETUP_SQPOLL so a kernel thread polls the submission queue and submissions are syscall-free. Each ring gets its own poll thread. Ignored for libaio / on Windows. Off by default. + /// io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle). Only used when is true; 0 = native default. public NativeStorageDevice(string filename, bool deleteOnClose = false, bool disableFileBuffering = true, long capacity = Devices.CAPACITY_UNSPECIFIED, int numCompletionThreads = 1, IoBackend ioBackend = IoBackend.Default, - ILogger logger = null) + ILogger logger = null, + int numIoContexts = 0, + int queueDepth = 0, + bool uringSqPoll = false, + int uringSqPollIdleMs = 0) : base(filename, EnsureParentDirectoryAndProbeSectorSize(filename), capacity) { Debug.Assert(numCompletionThreads >= 1); @@ -813,19 +1289,64 @@ public NativeStorageDevice(string filename, this.deleteOnClose = deleteOnClose; this.disableFileBuffering = disableFileBuffering; this.numCompletionThreadsConfig = numCompletionThreads < 1 ? 1 : numCompletionThreads; - // rings always track numCompletionThreads (1:1 drainer-to-ring binding). Each - // drainer blocks on its own ring inside QueueRunFor with a timeout, so a single - // drainer cannot cover multiple rings without starving any ring whose submitters - // produce completions while the drainer is parked on another ring. With per-thread - // submit affinity (pick_ring's thread_local index), every ring eventually receives - // submissions, so each ring must have its own drainer. For throughput scaling, - // callers should set numCompletionThreads >= expected submitter concurrency. - this.numIoContextsConfig = this.numCompletionThreadsConfig; + // The number of io_contexts (rings) is decoupled from the number of drainer threads. + // Each submitter maps to its own ring via the native pick_context thread-affinity, so + // giving the device more rings than concurrent submitters makes io_submit contention-free + // (no shared per-context aio ring/completion lock across unrelated submitters) and spreads + // completion posting across more rings. A small pool of numCompletionThreads drainers then + // range-drains contiguous slices of the rings. Any explicit value is clamped up to + // numCompletionThreads so every drainer owns at least one ring. + // + // Smart default when the caller leaves numIoContexts unset (<= 0): io_uring is ring-STARVED + // when rings < submitter concurrency — many submitters serialize on the per-ring submit lock + // (~3x slower). So default io_uring to a hardware-aware ring count that covers typical + // submitter concurrency (2x cores, capped at 64 to bound ring memory at ~400 KB/ring). + // libaio is ring-count-neutral (its kernel io_context mutex is cheap) and its + // io-contexts x queue-depth draws from the global fs.aio-max-nr budget, so it keeps the + // conservative rings = drainers default. + int defaultIoContexts = ioBackend == IoBackend.Uring + ? Math.Max(this.numCompletionThreadsConfig, Math.Min(2 * Environment.ProcessorCount, 64)) + : this.numCompletionThreadsConfig; + int requestedIoContexts = numIoContexts <= 0 ? defaultIoContexts : numIoContexts; + if (requestedIoContexts < this.numCompletionThreadsConfig) + requestedIoContexts = this.numCompletionThreadsConfig; + this.numIoContextsConfig = requestedIoContexts; + // Per-ring kernel submission depth D (maxEvents). 0 => DefaultQueueDepth at creation. + // Orthogonal to ring count (numIoContexts) and aggregate throttle; see ResolveQueueDepth. + this.numQueueDepthConfig = queueDepth > 0 ? queueDepth : 0; this.ioBackendConfig = ioBackend; + this.uringSqPollConfig = uringSqPoll; + this.uringSqPollIdleMsConfig = uringSqPollIdleMs > 0 ? uringSqPollIdleMs : 0; this.logger = logger; - ThrottleLimit = 120; + // Default aggregate in-flight throttle. Unlike the managed in-box devices (which cap at 120), + // the native device is built for deep NVMe queues, so it defaults to DefaultThrottleLimit (4096). + // The factory only overrides this when --device-throttle-limit is set (> 0); leaving it here means + // PerThreadLimit()/init resolve the 4096 default (their `ThrottleLimit > 0 ? ... : DefaultThrottleLimit` + // fallback is otherwise unreachable). No external consumer reads ThrottleLimit — Throttle() uses the + // sharded PerThreadLimit() path — so setting it to the intended default here is safe. + ThrottleLimit = DefaultThrottleLimit; _callbackDelegate = _callback; + + // In-flight accounting is sharded per submitter thread to avoid the global cache-line + // contention at high IOPS. The counter array is allocated up front (small, + // a few KB) because Throttle() may run before the first IO creates the native device. + shardInFlight = new ShardCounter[NumShards]; + shardIndex = new ThreadLocal(AssignShard); + + // Per-shard free-list of completion slots, each pre-populated with its shard's contiguous block of + // slot offsets into results[]. Allocated up front (not lazily per shard) so a completion drainer can + // safely return a slot to its owning shard's list without racing that list's construction. See + // shardFreeSlots. results[] itself is allocated lazily on first IO (EnsureNativeDeviceCreated). + shardFreeSlots = new ConcurrentQueue[NumShards]; + for (int s = 0; s < NumShards; s++) + { + var freeList = new ConcurrentQueue(); + int baseOffset = s * SlotsPerShard; + for (int k = 0; k < SlotsPerShard; k++) + freeList.Enqueue(baseOffset + k); + shardFreeSlots[s] = freeList; + } } /// @@ -902,12 +1423,41 @@ void EnsureNativeDeviceCreated() nativeSegmentSizeBytes = sizeForNative; - // Capture the effective in-flight throttle once, here, where the ring depth is also fixed — - // ThrottleLimit has been applied by the factory before the first IO. Throttle() then reads this - // field directly instead of re-clamping on every spin-loop iteration. - effectiveThrottleLimit = Math.Min(ThrottleLimit, MaxThrottle); + // Capture the effective aggregate in-flight throttle T once, here, where ring count N and + // depth D are also fixed — ThrottleLimit has been applied by the factory before the first IO. + // Clean split (one duty each): N = io-contexts (ring count), D = queue-depth (per-ring kernel + // depth), T = throttle-limit (aggregate software backpressure). T is capped at the kernel + // capacity N*D so aggregate in-flight can never exceed the rings (the correctness invariant + // that prevents the ring-full submit spin). Throttle() reads effectiveThrottleLimit directly. + int ringCount = numIoContextsConfig < 1 ? 1 : numIoContextsConfig; + int requestedThrottle = ThrottleLimit > 0 ? ThrottleLimit : DefaultThrottleLimit; + int ringDepth = ResolveQueueDepth(); + + // libaio io_setup PERMANENTLY reserves ringCount*ringDepth events from the GLOBAL fs.aio-max-nr + // budget at creation, used or not; the DefaultQueueDepth ceiling (right for io_uring's per-ring + // mmap SQ) over-reserves for libaio and, with many coexisting devices (e.g. cluster nodes), + // exhausts a stock 65536 budget => io_setup EAGAIN. When queue-depth is left at the default, + // size the libaio reservation to the throttle share (ringCount*reservation >= throttle) so the + // full aggregate throttle is preserved at no IOPS cost while the per-device reservation drops. + // The per-device fs.aio-max-nr ceiling applied inside runs last and can reduce it further. + if ((ioBackendConfig == IoBackend.Libaio || ioBackendConfig == IoBackend.Default) && numQueueDepthConfig <= 0) + ringDepth = ResolveLibaioReservationDepth(ringCount, requestedThrottle, ringDepth); + + WarnIfLibaioAioBudgetExceeded(ringCount, ringDepth); + long kernelCapacity = (long)ringCount * ringDepth; + if (requestedThrottle > kernelCapacity) + { + logger?.LogWarning( + "NativeStorageDevice: throttle-limit ({throttle}) exceeds kernel capacity io-contexts*queue-depth ({n}*{d}={cap}); capping aggregate in-flight at that capacity.", + requestedThrottle, ringCount, ringDepth, kernelCapacity); + requestedThrottle = (int)kernelCapacity; + } + effectiveThrottleLimit = requestedThrottle; - var newDevice = NativeDevice_CreateWithBackend(filename, false, disableFileBuffering, deleteOnClose, (int)ioBackendConfig, sizeForNative, OmitSegmentIdFromFileName, numIoContextsConfig, ComputeNativeRingDepth()); + // SQPOLL is an io_uring-only submit-side optimization; never pass it to libaio (the native + // libaio/default handlers accept and ignore it, but keep the managed intent explicit). + int uringSqPollArg = (uringSqPollConfig && ioBackendConfig == IoBackend.Uring) ? 1 : 0; + IntPtr newDevice = NativeDevice_CreateWithBackend(filename, false, disableFileBuffering, deleteOnClose, (int)ioBackendConfig, sizeForNative, OmitSegmentIdFromFileName, numIoContextsConfig, ringDepth, uringSqPollArg, uringSqPollIdleMsConfig); if (newDevice == IntPtr.Zero) { var nativeMessage = GetNativeLastError(); @@ -926,77 +1476,137 @@ void EnsureNativeDeviceCreated() : "Verify the native library matches the requested backend.")); } - ulong actualSegmentSize = NativeDevice_GetSegmentSize(newDevice); - if (actualSegmentSize != sizeForNative) + // Exception-safe initialization: newDevice is created but not yet published to the + // nativeDevice field, so any throw between here and the Volatile.Write below would + // otherwise (a) leak the native handle — Dispose observes nativeDevice == Zero and skips + // NativeDevice_Destroy — and (b) leave partially-started drainer threads in + // completionThreads (with null slots) that a later Dispose would NRE on while joining. + // The region starts at handle creation so it also covers the ABI-probe P/Invokes below, + // which throw EntryPointNotFoundException against an older native library. + // On any failure, stop whatever drainers were started, dispose the token, reset the + // partial fields, destroy the handle, and rethrow. + try { - NativeDevice_Destroy(newDevice); - throw new TsavoriteException( - $"Native device segment size mismatch: requested {sizeForNative}, native returned {actualSegmentSize}. " + - "This indicates an ABI mismatch between the loaded native_device library and the managed wrapper. " + - "Ensure libnative_device.so matches the current build."); - } + ulong actualSegmentSize = NativeDevice_GetSegmentSize(newDevice); + if (actualSegmentSize != sizeForNative) + { + throw new TsavoriteException( + $"Native device segment size mismatch: requested {sizeForNative}, native returned {actualSegmentSize}. " + + "This indicates an ABI mismatch between the loaded native_device library and the managed wrapper. " + + "Ensure libnative_device.so matches the current build."); + } - uint nativeSectorSize = NativeDevice_sector_size(newDevice); - if (nativeSectorSize != SectorSize) - { - // Both sides (managed probe in EnsureParentDirectoryAndProbeSectorSize, - // native probe in NativeDeviceImpl's field initializer) go through the - // same ProbeDioAlignment routine on the same filename with the parent - // directory pre-materialised, so the two values are guaranteed to agree - // on every well-formed host. A drift here is a real ABI / loaded-library - // mismatch — e.g. the shipped libnative_device.so was rebuilt from a - // different branch than the managed wrapper, or the host kernel changed - // STATX_DIOALIGN semantics between the two calls. Hard-fail so it is - // caught at the first I/O rather than silently mis-aligning every write. - NativeDevice_Destroy(newDevice); - throw new TsavoriteException( - $"Native device sector-size mismatch on '{filename}': managed wrapper probed {SectorSize} bytes but the kernel reports {nativeSectorSize} bytes for the actual file. " + - "The most likely cause is a stale libnative_device.so or a managed/native version skew. " + - "Rebuild the native library from this branch (libs/storage/Tsavorite/cc) and reinstall the resulting binary into libs/storage/Tsavorite/cs/src/core/Device/runtimes//native/."); - } + uint nativeSectorSize = NativeDevice_sector_size(newDevice); + if (nativeSectorSize != SectorSize) + { + // Both sides (managed probe in EnsureParentDirectoryAndProbeSectorSize, + // native probe in NativeDeviceImpl's field initializer) go through the + // same ProbeDioAlignment routine on the same filename with the parent + // directory pre-materialised, so the two values are guaranteed to agree + // on every well-formed host. A drift here is a real ABI / loaded-library + // mismatch — e.g. the shipped libnative_device.so was rebuilt from a + // different branch than the managed wrapper, or the host kernel changed + // STATX_DIOALIGN semantics between the two calls. Hard-fail so it is + // caught at the first I/O rather than silently mis-aligning every write. + throw new TsavoriteException( + $"Native device sector-size mismatch on '{filename}': managed wrapper probed {SectorSize} bytes but the kernel reports {nativeSectorSize} bytes for the actual file. " + + "The most likely cause is a stale libnative_device.so or a managed/native version skew. " + + "Rebuild the native library from this branch (libs/storage/Tsavorite/cc) and reinstall the resulting binary into libs/storage/Tsavorite/cs/src/core/Device/runtimes//native/."); + } + + if (results == null) results = new NativeResult[MaxResults]; - if (results == null) results = new NativeResult[MaxResults]; + // NativeDevice_QueueRun doubles as the platform capability probe: a negative result + // means the backend has no drainable completion queue, so no drainer is started. Only + // the Windows IOCP backend answers that way (completions arrive on threadpool threads); + // the Linux backends report the number of completions reaped. A zero timeout never + // blocks — libaio passes a zero io_getevents timeout and io_uring reads the completion + // queue in user space — so neither can answer with a transient error here. + if (NativeDevice_QueueRun(newDevice, 0) >= 0) + { + try + { + _ = NativeDevice_NumIoContexts(newDevice); + _ = NativeDevice_QueueRunFor(newDevice, 0, 0); + _ = NativeDevice_TryCompleteMine(newDevice); + } + catch (EntryPointNotFoundException ex) + { + throw new TsavoriteException( + "Loaded libnative_device.so/dll is missing the sharded-ABI exports " + + "NativeDevice_NumIoContexts / NativeDevice_QueueRunFor / NativeDevice_TryCompleteMine. " + + "The shared library predates the multi-io-context change and must be rebuilt from this branch " + + "(libs/storage/Tsavorite/cc) and the resulting binary installed to " + + "libs/storage/Tsavorite/cs/src/core/Device/runtimes//native/.", ex); + } + + completionThreadToken = new(); + int actualIoContexts = NativeDevice_NumIoContexts(newDevice); + if (actualIoContexts < 1) actualIoContexts = 1; + numRingsActual = actualIoContexts; + if (uringSqPollConfig && ioBackendConfig == IoBackend.Uring) + logger?.LogInformation( + "NativeStorageDevice: io_uring SQPOLL enabled (rings={rings}, one kernel submission-poll thread per ring, sq_thread_idle={idle}). Submissions are syscall-free while the poll thread is awake.", + actualIoContexts, uringSqPollIdleMsConfig > 0 ? $"{uringSqPollIdleMsConfig}ms" : "native-default"); + // Partition the io_contexts (rings) across a small pool of drainer threads. When + // actualIoContexts == numCompletionThreadsConfig this reduces to a 1:1 + // ring-to-drainer binding; when there are more rings than drainers (to de-contend + // io_submit), each drainer range-drains a contiguous slice so every ring is still + // reaped promptly. Rings are split as evenly as possible; the first `remainder` + // drainers get one extra ring. + int numDrainers = numCompletionThreadsConfig; + if (numDrainers > actualIoContexts) numDrainers = actualIoContexts; + completionThreads = new Thread[numDrainers]; + int baseCount = actualIoContexts / numDrainers; + int remainder = actualIoContexts % numDrainers; + int nextStart = 0; + for (int i = 0; i < numDrainers; i++) + { + int startCtx = nextStart; + int count = baseCount + (i < remainder ? 1 : 0); + nextStart += count; + var drainer = new Thread(() => CompletionWorker(startCtx, count)) + { + IsBackground = true + }; + // Publish the slot only once the thread is running: Join() on an unstarted + // thread throws ThreadStateException, which would escape the cleanup below and + // leak the native handle if Start() failed under resource pressure. + drainer.Start(); + completionThreads[i] = drainer; + } + } - if (NativeDevice_QueueRun(newDevice, 0) >= 0) + // Publish last: a reader observing nativeDevice != IntPtr.Zero is guaranteed to + // see a fully-initialised handle with completion threads already running. + Volatile.Write(ref nativeDevice, newDevice); + } + catch { try { - _ = NativeDevice_NumIoContexts(newDevice); - _ = NativeDevice_QueueRunFor(newDevice, 0, 0); + // Stop any drainers that were started. Pre-publish they are spin-yielding on the + // still-null nativeDevice field (NativeDevice_QueueRunFor null-guards and returns -1), + // so cancellation is observed promptly and Join returns quickly; null slots (a thread + // that was never created/started) are skipped. + if (completionThreads != null) + { + completionThreadToken?.Cancel(); + foreach (var t in completionThreads) t?.Join(); + completionThreads = null; + } + completionThreadToken?.Dispose(); + completionThreadToken = null; + numRingsActual = 0; } - catch (EntryPointNotFoundException ex) + finally { + // Free the handle even if drainer teardown throws: nativeDevice was never published, + // so Dispose() would skip the destroy and leak the fd and its kernel rings. NativeDevice_Destroy(newDevice); - throw new TsavoriteException( - "Loaded libnative_device.so/dll is missing the sharded-ABI exports " + - "NativeDevice_NumIoContexts / NativeDevice_QueueRunFor. The shared library " + - "predates the multi-io-context change and must be rebuilt from this branch " + - "(libs/storage/Tsavorite/cc) and the resulting binary installed to " + - "libs/storage/Tsavorite/cs/src/core/Device/runtimes//native/.", ex); - } - - completionThreadToken = new(); - int actualIoContexts = NativeDevice_NumIoContexts(newDevice); - if (actualIoContexts < 1) actualIoContexts = 1; - // We pass numCompletionThreadsConfig to the native ctor as num_io_contexts; - // the native side may clamp at 1 if it received 0 or negative, but otherwise - // honors it. So actualIoContexts should equal numCompletionThreadsConfig. Each - // drainer is bound 1:1 to its own ring via QueueRunFor(ctxIdx, ...). - completionThreads = new Thread[actualIoContexts]; - for (int i = 0; i < actualIoContexts; i++) - { - int ctxIdx = i; - completionThreads[i] = new Thread(() => CompletionWorker(ctxIdx)) - { - IsBackground = true - }; - completionThreads[i].Start(); } + throw; } - - // Publish last: a reader observing nativeDevice != IntPtr.Zero is guaranteed to - // see a fully-initialised handle with completion threads already running. - Volatile.Write(ref nativeDevice, newDevice); } } @@ -1009,16 +1619,10 @@ void EnsureNativeDeviceCreated() /// public override void Reset() { - if (Volatile.Read(ref disposedFlag) != 0) return; // Lease the native handle (same protocol as ReadAsync/WriteAsync) so a concurrent - // Dispose() cannot drain numPending to its int.MinValue poison and free the handle - // while we are inside the native call. A <= 0 result means Dispose already poisoned; - // restore it and no-op. - if (Interlocked.Increment(ref numPending) <= 0) - { - Interlocked.Decrement(ref numPending); - return; - } + // Dispose() cannot free it while we are inside the native call. A false result means + // disposal has begun; no-op. + if (!TryLease(out int shard)) return; try { // No-op if the native device has not been created yet (no handles to reset). @@ -1028,7 +1632,7 @@ public override void Reset() } finally { - Interlocked.Decrement(ref numPending); + ReleaseLease(shard); } } @@ -1081,55 +1685,74 @@ public override void ReadAsync(int segmentId, ulong sourceAddress, // operations is negligible vs the syscall itself. ThrowIfMisaligned(sourceAddress, readLength, destinationAddress, nameof(ReadAsync)); - int offset; - while (!freeResults.TryDequeue(out offset)) + // Sharded, contention-free slot + in-flight accounting. The slot is rented from this submitter's + // shard free-list and returned only when the IO completes (in _callback or an error path below), so + // a slow IO's slot is never reused while it is still in flight — the correctness the counter-ring it + // replaced could not provide under out-of-order completions. The SubmitToShard bump is the in-flight + // "lease" the Dekker-style Dispose fence and Throttle() observe. See RentSlot / shardFreeSlots. + int shard = GetShard(); + int offset = RentSlot(shard); + SubmitToShard(shard); + // Fence against a concurrent Dispose that began after the EnsureNativeDeviceCreated check above: + // if disposal is now visible, balance the submit bump and route the error callback rather than + // touching a handle Dispose may be about to free. (Dekker-style: Dispose sets the flag then drains + // in-flight, so either it observes this bump and waits, or we observe the flag here and back out.) + if (Volatile.Read(ref disposedFlag) != 0) { - if (resultOffset < MaxResults) - { - offset = Interlocked.Increment(ref resultOffset) - 1; - if (offset < MaxResults) break; - } - Thread.Yield(); + CompleteShard(shard); + ReturnSlot(offset); + callback(uint.MaxValue, 0, context, ioException: default); + return; } ref var result = ref results[offset]; result.context = context; result.callback = callback; + // Second, independently balanced in-flight lease covering the native call itself. The IO's own + // bump above is dropped by _callback, which runs on a drainer thread and can fire before this + // frame has left native code; without this lease Dispose could observe zero in-flight and free + // the device under a submitter still inside it. Same role as TryLease on the non-IO entry points. + EnterNativeCall(shard); try { - if (Interlocked.Increment(ref numPending) <= 0) - throw new Exception("Cannot operate on disposed device"); - int _result = NativeDevice_ReadAsync(nativeDevice, ((ulong)segmentId << segmentSizeBits) | sourceAddress, destinationAddress, readLength, _callbackDelegate, (IntPtr)offset); - - if (_result != 0) - throw new IOException($"Error reading from log file (status {_result}){FormatNativeError()}", _result); - } - catch (IOException e) - { - logger?.LogCritical(e, $"{nameof(ReadAsync)}"); try { - callback((uint)(e.HResult & 0x0000FFFF), 0, context, ioException: e); - } - finally - { - freeResults.Enqueue(offset); - Interlocked.Decrement(ref numPending); + int _result = NativeDevice_ReadAsync(nativeDevice, ((ulong)segmentId << segmentSizeBits) | sourceAddress, destinationAddress, readLength, _callbackDelegate, (IntPtr)offset); + + if (_result != 0) + throw new IOException($"Error reading from log file (status {_result}){FormatNativeError()}", _result); } - } - catch (Exception e) - { - logger?.LogCritical(e, $"{nameof(ReadAsync)}"); - try + catch (IOException e) { - callback(uint.MaxValue, 0, context, ioException: e); + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); + try + { + callback((uint)(e.HResult & 0x0000FFFF), 0, context, ioException: e); + } + finally + { + CompleteShard(shard); + ReturnSlot(offset); + } } - finally + catch (Exception e) { - freeResults.Enqueue(offset); - Interlocked.Decrement(ref numPending); + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); + try + { + callback(uint.MaxValue, 0, context, ioException: e); + } + finally + { + CompleteShard(shard); + ReturnSlot(offset); + } } } + finally + { + ExitNativeCall(shard); + } } /// @@ -1150,74 +1773,64 @@ public override unsafe void WriteAsync(IntPtr sourceAddress, // whichever upper-layer staging buffer is misaligned. ThrowIfMisaligned(destinationAddress, numBytesToWrite, sourceAddress, nameof(WriteAsync)); - int offset; - while (!freeResults.TryDequeue(out offset)) + // Sharded slot + in-flight accounting; see ReadAsync for the full rationale. + int shard = GetShard(); + int offset = RentSlot(shard); + SubmitToShard(shard); + if (Volatile.Read(ref disposedFlag) != 0) { - if (resultOffset < MaxResults) - { - offset = Interlocked.Increment(ref resultOffset) - 1; - if (offset < MaxResults) break; - } - Thread.Yield(); + CompleteShard(shard); + ReturnSlot(offset); + callback(uint.MaxValue, 0, context, ioException: default); + return; } ref var result = ref results[offset]; result.context = context; result.callback = callback; + // Native-call lease; see ReadAsync for the rationale. + EnterNativeCall(shard); try { - var newPending = Interlocked.Increment(ref numPending); - if (newPending <= 0) - throw new Exception("Cannot operate on disposed device"); - if (s_instrument) + try { - Interlocked.Increment(ref submitCount); - var prevPeak = peakNumPending; - while (newPending > prevPeak) + int _result = NativeDevice_WriteAsync(nativeDevice, sourceAddress, ((ulong)segmentId << segmentSizeBits) | destinationAddress, numBytesToWrite, _callbackDelegate, (IntPtr)offset); + + if (_result != 0) { - var actual = Interlocked.CompareExchange(ref peakNumPending, newPending, prevPeak); - if (actual == prevPeak) break; - prevPeak = actual; + throw new IOException($"Error writing to log file (status {_result}){FormatNativeError()}", _result); } } - long ts0 = s_instrument ? Stopwatch.GetTimestamp() : 0; - int _result = NativeDevice_WriteAsync(nativeDevice, sourceAddress, ((ulong)segmentId << segmentSizeBits) | destinationAddress, numBytesToWrite, _callbackDelegate, (IntPtr)offset); - if (s_instrument) - { - var elapsed = Stopwatch.GetTimestamp() - ts0; - Interlocked.Add(ref submitNanos, (long)(elapsed * 1_000_000_000.0 / Stopwatch.Frequency)); - } - - if (_result != 0) + catch (IOException e) { - throw new IOException($"Error writing to log file (status {_result}){FormatNativeError()}", _result); - } - } - catch (IOException e) - { - logger?.LogCritical(e, $"{nameof(WriteAsync)}"); - try - { - callback((uint)(e.HResult & 0x0000FFFF), 0, context, ioException: e); + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); + try + { + callback((uint)(e.HResult & 0x0000FFFF), 0, context, ioException: e); + } + finally + { + CompleteShard(shard); + ReturnSlot(offset); + } } - finally + catch (Exception e) { - freeResults.Enqueue(offset); - Interlocked.Decrement(ref numPending); + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); + try + { + callback(uint.MaxValue, 0, context, ioException: e); + } + finally + { + CompleteShard(shard); + ReturnSlot(offset); + } } } - catch (Exception e) + finally { - logger?.LogCritical(e, $"{nameof(WriteAsync)}"); - try - { - callback(uint.MaxValue, 0, context, ioException: e); - } - finally - { - freeResults.Enqueue(offset); - Interlocked.Decrement(ref numPending); - } + ExitNativeCall(shard); } } @@ -1229,24 +1842,27 @@ public override void RemoveSegment(int segment) { if (Volatile.Read(ref disposedFlag) != 0) return; // Lease the native handle so a concurrent Dispose() can't free it mid-call. - if (Interlocked.Increment(ref numPending) <= 0) - { - Interlocked.Decrement(ref numPending); - return; - } - try + if (TryLease(out int shard)) { - var dev = Volatile.Read(ref nativeDevice); - if (dev != IntPtr.Zero) + try + { + var dev = Volatile.Read(ref nativeDevice); + if (dev != IntPtr.Zero) + { + // Native owns the open handle; let it close+unlink. + NativeDevice_RemoveSegment(dev, (ulong)segment); + return; + } + } + finally { - // Native owns the open handle; let it close+unlink. - NativeDevice_RemoveSegment(dev, (ulong)segment); - return; + ReleaseLease(shard); } } - finally + else { - Interlocked.Decrement(ref numPending); + // Disposal began — match the disposed-at-entry behavior above and no-op. + return; } // No native handle yet — delete the on-disk segment file directly so callers // observe the same semantics as LocalStorageDevice / RandomAccessLocalStorageDevice @@ -1269,16 +1885,22 @@ public override void RemoveSegmentAsync(int segment, AsyncCallback callback, IAs /// /// Close device. Shutdown ordering matters: any in-flight IOs must complete first so the - /// numPending CAS terminates; the completion threads must exit BEFORE we destroy the native + /// in-flight drain terminates; the completion threads must exit BEFORE we destroy the native /// device, otherwise they can dereference a freed io_uring/libaio ring inside /// . /// /// /// Idempotent — multiple calls are safe; only the first does work. /// - /// User IO callbacks fire on completion-worker threads. Dispose() cannot run on one of - /// those threads, because joining the caller would deadlock — we detect and throw - /// in that case. + /// User IO callbacks fire either on a completion-worker (drainer) thread or inline on a + /// submitter thread that reaps its own completions via / + /// TryCompleteMine (the default affine inline-drain path). Dispose() must NOT be called from + /// within any such callback: the in-flight drain below waits for that very callback's completion + /// bump (issued in _callback's finally), so a self-dispose would spin forever. The + /// drainer-thread case is detected and thrown as ; the + /// inline-submitter-thread case cannot be cheaply detected on the hot completion path, so it is + /// the caller's contract (matching the IDevice lifecycle contract) not to dispose the device + /// from inside an IO completion callback — post the disposal to a separate thread instead. /// /// /// Worst-case shutdown stall is bounded by the duration of the longest in-flight user @@ -1311,19 +1933,39 @@ public override void Dispose() // Idempotent: second and subsequent calls short-circuit. Setting the flag here gates // the late P/Invoke entry points (TryComplete, GetFileSize, Reset, RemoveSegment) on - // their first line; the numPending lease they then take closes the race where this - // drain poisons and frees the handle between that check and the native call. + // their first line; the per-shard lease they then take closes the race where this + // drain frees the handle between that check and the native call. if (Interlocked.Exchange(ref disposedFlag, 1) != 0) return; - // Drain in-flight ops by poisoning numPending to int.MinValue once it hits 0. Submit - // paths fail their Interlocked.Increment(numPending) <= 0 check and route through the - // error callback; the _callback decrement in the success path runs in `finally` after - // the user callback, so by the time we observe numPending == 0 all completions are done. - while (numPending >= 0) + // Drain in-flight ops: wait until every shard's in-flight count returns to zero. disposedFlag was + // published above (full barrier); submit/lease paths bump their shard (full barrier) then re-check + // the flag, so — Dekker-style — either they observe disposal and back out without touching the + // handle, or this drain observes their bump and waits. The in-flight decrement for an accepted IO + // runs in _callback's `finally` after the user callback, so once in-flight reaches 0 all completions + // (and their user callbacks) have finished. + // + // Bounded, because in-flight only returns to zero if the kernel completes every accepted IO. A + // completion that never arrives (a stalled device or driver, a dropped CQE, an io_uring ring whose + // SQPOLL thread has died) would otherwise spin here forever, burning a core and reporting nothing. + // The deadline sits orders of magnitude above any legitimate drain, so reaching it means completions + // are lost rather than slow. Proceeding is then safe: the drainers are cancelled and joined before + // the handle is freed, the separate lease drain below stops this deadline from freeing the handle + // under a native call still in progress, and NativeDevice_Destroy cancels or waits for whatever the + // kernel still owns. + var drainStart = Stopwatch.StartNew(); + var drainSpin = new SpinWait(); + while (TotalInFlight() != 0) { - Interlocked.CompareExchange(ref numPending, int.MinValue, 0); - Thread.Yield(); + if (drainStart.ElapsedMilliseconds >= DisposeDrainTimeoutMs) + { + logger?.LogError( + "NativeStorageDevice.Dispose() timed out after {timeoutMs}ms with {inFlight} in-flight IO(s) whose completions " + + "were never delivered by the kernel. Proceeding with teardown; their buffers and contexts are leaked.", + DisposeDrainTimeoutMs, TotalInFlight()); + break; + } + drainSpin.SpinOnce(sleep1Threshold: DisposeDrainSleepThreshold); } // Cancel and Join every completion thread, then destroy the native device. @@ -1335,14 +1977,14 @@ public override void Dispose() { completionThreadToken.Cancel(); // Wake every blocked completion drainer by submitting a no-op IO to each - // io_context. The drainer is otherwise sleeping in NativeDevice_QueueRunFor - // waiting for completion events; the wake-up causes the syscall to return - // promptly so the cancellation token can be observed on the next loop - // iteration. Best-effort: on submit failure the drainer still wakes when + // io_context. A drainer parks in NativeDevice_QueueRunFor on the first ring of + // its range, so wake every ring [0, numRingsActual): waking a ring that no + // drainer is currently parked on is harmless (the no-op event is reaped on the + // next drain pass). Best-effort: on submit failure the drainer still wakes when // its QueueRunFor timeout fires. - for (int i = 0; i < completionThreads.Length; i++) + for (int i = 0; i < numRingsActual; i++) _ = NativeDevice_WakeCompletionWorker(nativeDevice, i); - foreach (var t in completionThreads) t.Join(); + foreach (var t in completionThreads) t?.Join(); completionThreadToken.Dispose(); completionThreads = null; } @@ -1350,6 +1992,34 @@ public override void Dispose() var dev = Interlocked.Exchange(ref nativeDevice, IntPtr.Zero); if (dev != IntPtr.Zero) { + // Destroying the handle is gated on the lease counter rather than on the in-flight drain + // above. That drain has a deadline because a completion can be lost forever; a lease instead + // means a thread is executing inside native code right now. TryComplete / TryCompleteMine + // hold one across a native call that dispatches user callbacks inline, so a lease is bounded + // only by user code — letting the lost-completion deadline free the handle would pull the + // rings and locks out from under a running native frame. No lease can be newly acquired once + // disposedFlag is published (every lease site re-checks it after publishing), so this waits + // only for calls that were already inside native code, and it is a single read in the normal + // case: leases are a subset of in-flight, so an in-flight drain that completed proves zero. + var leaseStart = Stopwatch.StartNew(); + var leaseSpin = new SpinWait(); + while (TotalLeases() != 0) + { + if (leaseStart.ElapsedMilliseconds >= DisposeLeaseDrainTimeoutMs) + { + // Leak the native device rather than free it: a native frame still owns its rings + // and locks. A leaked handle at teardown is bounded and diagnosable; freeing memory + // under a running frame is neither. + logger?.LogError( + "NativeStorageDevice.Dispose() timed out after {timeoutMs}ms with {leases} native call(s) still " + + "executing. Leaking the native device instead of destroying it, since freeing it would " + + "invalidate state those calls are still using.", + DisposeLeaseDrainTimeoutMs, TotalLeases()); + return; + } + leaseSpin.SpinOnce(sleep1Threshold: DisposeDrainSleepThreshold); + } + NativeDevice_Destroy(dev); // NativeDevice_Destroy runs log_.Close() under the C ABI firewall; if that threw // it was caught and recorded rather than crashing teardown. Surface it so a @@ -1364,24 +2034,47 @@ public override void Dispose() /// public override bool TryComplete() { - if (Volatile.Read(ref disposedFlag) != 0) return false; - // Lease the native handle so a concurrent Dispose() can't free it mid-call. The - // disposedFlag check above rejects the common post-dispose case without the interlocked - // cost; the lease closes the remaining race where Dispose poisons numPending between - // that check and the native call. - if (Interlocked.Increment(ref numPending) <= 0) - { - Interlocked.Decrement(ref numPending); + // Lease the native handle so a concurrent Dispose() can't free it mid-call. TryLease + // rejects the post-dispose case and closes the race where Dispose frees the handle + // between the flag check and the native call. + if (!TryLease(out int shard)) return false; + try + { + var dev = Volatile.Read(ref nativeDevice); + if (dev == IntPtr.Zero) + return false; + return NativeDevice_TryComplete(dev); + } + finally + { + ReleaseLease(shard); } + } + + /// + /// Drain only the calling thread's affine native context/ring (the one its submits land on), + /// instead of walking every context like . The inline submitter-thread + /// completion path (Tsavorite CompletePending / AsyncGetFromDisk throttle-wait) is the primary + /// reaper at high IOPS; polling just this thread's own context issues one io_getevents per poll + /// rather than one per context, cutting completion-drain syscalls (and the cross-context aio + /// ring-lock contention) by roughly the context count. All contexts stay covered because each + /// has sharing submitters and/or a dedicated completion (drainer) thread. + /// + public override bool TryCompleteMine() + { + if (!TryLease(out int shard)) + return false; try { var dev = Volatile.Read(ref nativeDevice); - return dev != IntPtr.Zero && NativeDevice_TryComplete(dev); + if (dev == IntPtr.Zero) + return false; + return NativeDevice_TryCompleteMine(dev); } finally { - Interlocked.Decrement(ref numPending); + ReleaseLease(shard); } } @@ -1390,7 +2083,7 @@ public override long GetFileSize(int segment) { if (Volatile.Read(ref disposedFlag) != 0) return 0; // Lease the native handle so a concurrent Dispose() can't free it mid-call. - if (Interlocked.Increment(ref numPending) > 0) + if (TryLease(out int shard)) { try { @@ -1400,13 +2093,9 @@ public override long GetFileSize(int segment) } finally { - Interlocked.Decrement(ref numPending); + ReleaseLease(shard); } } - else - { - Interlocked.Decrement(ref numPending); - } // No native handle yet (or disposed) — stat the on-disk segment file directly. Matches // LocalStorageDevice / RandomAccessLocalStorageDevice semantics where size is // observable before any IO has flowed through the device. Returns 0 for missing @@ -1547,41 +2236,91 @@ internal static uint ProbeSectorSize(string filename) } /// - /// Drain loop for one completion thread, bound 1:1 to ring shard . - /// Blocks in NativeDevice_QueueRunFor with a long timeout. Dispose() wakes blocked - /// workers via NativeDevice_WakeCompletionWorker rather than relying on the - /// timeout to fire. + /// Drain loop for one completion thread. The thread owns the contiguous range of ring + /// shards [startCtx, startCtx + ctxCount). For a single ring it blocks in + /// NativeDevice_QueueRunFor with a long timeout (single-ring fast path), and Dispose() + /// wakes it via NativeDevice_WakeCompletionWorker rather than relying on the timeout to + /// fire. For a range it never blocks on any one ring — blocking would hide completions on the + /// siblings for the whole timeout — so it polls every ring non-blocking (timeout 0), yields on a + /// brief idle, and sleeps 1 ms only after consecutive + /// idle passes; cancellation is observed within one pass, with no wake dependency. /// - void CompletionWorker(int ctxIdx) + void CompletionWorker(int startCtx, int ctxCount) { // Defense-in-depth: catch around the whole drain loop. _callback already swallows // all exceptions from the user callback (see its big comment), but if anything // else managed-side throws here (e.g. nativeDevice goes IntPtr.Zero mid-call // during a race with Dispose, or a P/Invoke marshalling exception), losing the - // drainer thread silently is catastrophic: no completions ever fire, numPending + // drainer thread silently is catastrophic: no completions ever fire, in-flight // grows unbounded, the next submitter spins forever in device.Throttle() and the // whole engine deadlocks. So if anything escapes, log it loudly and exit cleanly. try { + // Consecutive idle poll-passes for the multi-ring path below; reset on any drained event. + long idleSpins = 0; while (true) { if (completionThreadToken.IsCancellationRequested) break; - int rc = NativeDevice_QueueRunFor(nativeDevice, ctxIdx, CompletionWorkerTimeoutSecs); - if (rc == NativeCABIExceptionSentinel) + + if (ctxCount <= 1) + { + // Single ring: block directly on it with the timeout (single-ring fast path). + int rc = NativeDevice_QueueRunFor(nativeDevice, startCtx, CompletionWorkerTimeoutSecs); + if (rc == NativeCABIExceptionSentinel) + { + // The native drain threw and was firewalled by the C ABI guard (instead of + // unwinding across P/Invoke and terminating the process). Surface the message + // so it can be reported, then pause briefly to avoid a hot error loop if the + // fault is persistent. The drainer keeps running so Dispose can still proceed. + logger?.LogError("NativeStorageDevice completion drainer (startCtx={startCtx}) hit a native exception: {error}", startCtx, GetNativeLastError()); + Thread.Sleep(10); + } + Thread.Yield(); + continue; + } + + // Range of rings: poll each one non-blocking (timeout 0) so a completion on any + // ring in the range is reaped promptly. A blocking io_getevents parks on a SINGLE + // context, which would hide completions on the sibling rings for the whole timeout + // — fatal for the low-in-flight write/flush path (a stalled flush completion blocks + // the memory-bounded log). So this path NEVER blocks on one ring: under saturation + // every pass drains events and we re-poll immediately (max throughput, lowest + // latency); on a brief idle we yield; only on sustained idle do we sleep 1ms to + // release the core, and even then the next pass re-polls the ENTIRE range so no ring + // waits more than ~1ms. Dispose is observed within one pass (no Wake dependency here). + int drained = 0; + bool faulted = false; + for (int k = 0; k < ctxCount; k++) + { + int rc = NativeDevice_QueueRunFor(nativeDevice, startCtx + k, 0); + if (rc == NativeCABIExceptionSentinel) + faulted = true; + else if (rc > 0) + drained += rc; + } + if (faulted) { - // The native drain threw and was firewalled by the C ABI guard (instead of - // unwinding across P/Invoke and terminating the process). Surface the message - // so it can be reported, then pause briefly to avoid a hot error loop if the - // fault is persistent. The drainer keeps running so Dispose can still proceed. - logger?.LogError("NativeStorageDevice completion drainer (ctxIdx={ctxIdx}) hit a native exception: {error}", ctxIdx, GetNativeLastError()); + logger?.LogError("NativeStorageDevice completion drainer (startCtx={startCtx}, count={ctxCount}) hit a native exception: {error}", startCtx, ctxCount, GetNativeLastError()); Thread.Sleep(10); + idleSpins = 0; + } + else if (drained > 0) + { + idleSpins = 0; // stay hot: re-poll immediately + } + else if (++idleSpins < CompletionWorkerIdleSpinBudget) + { + Thread.Yield(); // brief idle: stay responsive, keep every ring visible + } + else + { + Thread.Sleep(1); // sustained idle: release the core (re-polls whole range on wake) } - Thread.Yield(); } } catch (Exception ex) { - logger?.LogCritical(ex, "NativeStorageDevice completion drainer (ctxIdx={ctxIdx}) terminated by unhandled exception", ctxIdx); + logger?.LogCritical(ex, "NativeStorageDevice completion drainer (startCtx={startCtx}) terminated by unhandled exception", startCtx); } } @@ -1589,5 +2328,29 @@ void CompletionWorker(int ctxIdx) // negligible; Dispose() does not rely on this firing because it submits a synthetic wake-up // event via NativeDevice_WakeCompletionWorker to unblock the worker immediately. const int CompletionWorkerTimeoutSecs = 1; + + // Multi-ring drainers (numIoContexts > numCompletionThreads) never block in a syscall (that + // would hide sibling rings). Instead they poll; after this many consecutive fully-idle passes + // they switch from Thread.Yield() to Thread.Sleep(1) to release the core. Under a saturated + // read workload the idle branch is never taken, so this only bounds CPU when the device is + // quiescent (e.g. between log-flush completions during load). + const long CompletionWorkerIdleSpinBudget = 1024; + + // Upper bound on Dispose()'s in-flight drain. A normal drain finishes in microseconds: the + // outstanding IOs are already queued in the kernel and the leases held across native calls are + // individually bounded. This only fires when completions are permanently lost, so it is set far + // above any legitimate drain to keep a degraded-but-live device from tripping it. + const int DisposeDrainTimeoutMs = 30_000; + + // Upper bound on Dispose()'s wait for in-progress native calls to return before the handle is destroyed. + // Unlike the in-flight drain this cannot be tripped by lost completions — only leases already inside + // native code are counted — so it fires only when a user callback dispatched inline by TryComplete / + // TryCompleteMine blocks indefinitely. Expiry leaks the handle rather than freeing it. + const int DisposeLeaseDrainTimeoutMs = 30_000; + + // Iterations Dispose()'s drain spins before SpinWait starts inserting Thread.Sleep(1). Keeps the + // common case (drain completes almost immediately) spin-fast while stopping a drain that runs to + // DisposeDrainTimeoutMs from pinning a core. + const int DisposeDrainSleepThreshold = 20; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs b/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs index c02d78d5217..af2a256f4e6 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs @@ -400,6 +400,9 @@ public virtual bool TryComplete() return true; } + /// + public virtual bool TryCompleteMine() => TryComplete(); + /// public virtual long GetFileSize(int segment) { diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device.so index 0699930bfa8..82f3d63c10f 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device_libaio.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device_libaio.so index 5fc56e8d290..ed1c8ceec4a 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device_libaio.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-arm64/native/libnative_device_libaio.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device.so index a1beb502c97..826d171bb82 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device_libaio.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device_libaio.so index 62472e10663..0a9f9e0c330 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device_libaio.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-arm64/native/libnative_device_libaio.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device.so index f7bbf06a464..8f9aef6d70a 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device_libaio.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device_libaio.so index 7f44d4d0f3b..3a5359b8467 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device_libaio.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-musl-x64/native/libnative_device_libaio.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device.so index ac19bb598a2..d839c901bdd 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device_libaio.so b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device_libaio.so index ffd34900bff..d77f91c2347 100755 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device_libaio.so and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/linux-x64/native/libnative_device_libaio.so differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.dll b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.dll index 39473fdd263..5ebdf690c3f 100644 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.dll and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.dll differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.pdb b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.pdb index f1e8cafedb7..c642cdd8b9d 100644 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.pdb and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-arm64/native/native_device.pdb differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.dll b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.dll index 5ee89968a6d..4b794314d49 100644 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.dll and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.dll differ diff --git a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.pdb b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.pdb index 6a6500223a2..66112b91e03 100644 Binary files a/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.pdb and b/libs/storage/Tsavorite/cs/src/core/Device/runtimes/win-x64/native/native_device.pdb differ diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactory.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactory.cs index b0292855a0d..d6aa95b58be 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactory.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactory.cs @@ -21,8 +21,8 @@ public class LocalStorageNamedDeviceFactory : INamedDeviceFactory readonly bool preallocateFile; readonly bool disableFileBuffering; readonly DeviceType deviceType; - readonly NativeStorageDevice.IoBackend ioBackend; readonly int numCompletionThreads; + readonly NativeDeviceOptions nativeDeviceOptions; readonly bool readOnly; readonly ILogger logger; @@ -34,20 +34,20 @@ public class LocalStorageNamedDeviceFactory : INamedDeviceFactory /// Whether file buffering (during write) is disabled (default of true requires aligned writes) /// Throttle limit (max number of pending I/Os) for this device instance. For DeviceType.LocalMemory (which has no device-wide throttle) it instead sets the per-ring capacity, rounded up to a power of two. /// Device type to use - /// For DeviceType.Native on Linux: which IO backend (libaio or io_uring) to use. Ignored otherwise. /// For DeviceType.Native on Linux: number of IO completion drain threads (default 1). Ignored otherwise. /// Whether files are opened as readonly /// Base name /// Logger - public LocalStorageNamedDeviceFactory(bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, int? throttleLimit = null, DeviceType deviceType = DeviceType.Default, NativeStorageDevice.IoBackend ioBackend = NativeStorageDevice.IoBackend.Default, int numCompletionThreads = 1, bool readOnly = false, string baseName = null, ILogger logger = null) + /// For DeviceType.Native on Linux: libaio / io_uring backend tuning (IO backend, ring count, per-ring queue depth, SQPOLL). Null (default) uses the backend-specific defaults. Ignored otherwise. See . + public LocalStorageNamedDeviceFactory(bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, int? throttleLimit = null, DeviceType deviceType = DeviceType.Default, int numCompletionThreads = 1, bool readOnly = false, string baseName = null, ILogger logger = null, NativeDeviceOptions nativeDeviceOptions = null) { this.preallocateFile = preallocateFile; this.deleteOnClose = deleteOnClose; this.disableFileBuffering = disableFileBuffering; this.throttleLimit = throttleLimit; this.deviceType = deviceType; - this.ioBackend = ioBackend; this.numCompletionThreads = numCompletionThreads; + this.nativeDeviceOptions = nativeDeviceOptions; this.readOnly = readOnly; this.baseName = baseName; this.logger = logger; @@ -74,10 +74,10 @@ public IDevice Get(FileDescriptor fileInfo) deleteOnClose: deleteOnClose, disableFileBuffering: disableFileBuffering, readOnly: readOnly, - ioBackend: ioBackend, numCompletionThreads: numCompletionThreads, - localMemoryRingCapacity: localMemoryRingCapacity, - logger: logger); + logger: logger, + nativeDeviceOptions: nativeDeviceOptions, + localMemoryDeviceOptions: new LocalMemoryDeviceOptions { RingCapacity = localMemoryRingCapacity }); if (throttleLimit.HasValue) { device.ThrottleLimit = throttleLimit.Value; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactoryCreator.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactoryCreator.cs index 9d57e6f3a50..04fbed6c537 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactoryCreator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/LocalStorageNamedDeviceFactoryCreator.cs @@ -15,8 +15,8 @@ public class LocalStorageNamedDeviceFactoryCreator : INamedDeviceFactoryCreator readonly int? throttleLimit; readonly bool disableFileBuffering; readonly DeviceType deviceType; - readonly NativeStorageDevice.IoBackend ioBackend; readonly int numCompletionThreads; + readonly NativeDeviceOptions nativeDeviceOptions; readonly bool readOnly; readonly ILogger logger; @@ -28,26 +28,26 @@ public class LocalStorageNamedDeviceFactoryCreator : INamedDeviceFactoryCreator /// Whether file buffering (during write) is disabled (default of true requires aligned writes) /// Throttle limit (max number of pending I/Os) for this device instance. For DeviceType.LocalMemory (which has no device-wide throttle) it instead sets the per-ring capacity, rounded up to a power of two. /// Device type - /// For DeviceType.Native on Linux: which IO backend (libaio or io_uring) to use. Ignored otherwise. /// For DeviceType.Native on Linux: number of IO completion drain threads (default 1). Ignored otherwise. /// Whether files are opened as readonly /// Logger - public LocalStorageNamedDeviceFactoryCreator(bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, int? throttleLimit = null, DeviceType deviceType = DeviceType.Default, NativeStorageDevice.IoBackend ioBackend = NativeStorageDevice.IoBackend.Default, int numCompletionThreads = 1, bool readOnly = false, ILogger logger = null) + /// For DeviceType.Native on Linux: libaio / io_uring backend tuning (IO backend, ring count, per-ring queue depth, SQPOLL). Null (default) uses the backend-specific defaults. Ignored otherwise. See . + public LocalStorageNamedDeviceFactoryCreator(bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, int? throttleLimit = null, DeviceType deviceType = DeviceType.Default, int numCompletionThreads = 1, bool readOnly = false, ILogger logger = null, NativeDeviceOptions nativeDeviceOptions = null) { this.preallocateFile = preallocateFile; this.deleteOnClose = deleteOnClose; this.disableFileBuffering = disableFileBuffering; this.throttleLimit = throttleLimit; this.deviceType = deviceType; - this.ioBackend = ioBackend; this.numCompletionThreads = numCompletionThreads; + this.nativeDeviceOptions = nativeDeviceOptions; this.readOnly = readOnly; this.logger = logger; } public INamedDeviceFactory Create(string baseName) { - return new LocalStorageNamedDeviceFactory(preallocateFile, deleteOnClose, disableFileBuffering, throttleLimit, deviceType, ioBackend, numCompletionThreads, readOnly, baseName, logger); + return new LocalStorageNamedDeviceFactory(preallocateFile, deleteOnClose, disableFileBuffering, throttleLimit, deviceType, numCompletionThreads, readOnly, baseName, logger, nativeDeviceOptions); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs index 428d89fcf77..fcf1e70faf3 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs @@ -80,7 +80,7 @@ internal void InternalCompletePendingRequests completedOutputs) where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - _ = hlogBase.TryComplete(); + _ = hlogBase.TryCompleteMine(); if (sessionFunctions.Ctx.readyResponses.Count == 0) return; diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.OriginReturn.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.OriginReturn.cs index 4ee5e90979b..f58247c6810 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.OriginReturn.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.OriginReturn.cs @@ -85,10 +85,10 @@ internal sealed class Bucket } /// - /// Per (pool, thread) shard owning one per size-class. Finalizable so that a - /// thread which dies with buffers still cached releases those buffers' budget permits (there is no - /// thread-exit callback). Self-clears its heavy references on seal so a lingering thread-static slot on - /// another thread roots nothing large. + /// Per (pool, thread) shard owning one per size-class, and the byte ceiling those + /// buckets share (see ). Finalizable so that a thread which dies with buffers still + /// cached releases those buffers' budget permits (there is no thread-exit callback). Self-clears its heavy + /// references on seal so a lingering thread-static slot on another thread roots nothing large. /// internal sealed class ThreadShard { @@ -100,9 +100,19 @@ internal sealed class ThreadShard internal int state; // Alive / Sealed internal int drainedOnce; // CAS 0->1: arbitrates explicit seal vs. finalization (drain at most once) - internal ThreadShard(SectorAlignedBufferPool pool, int numClasses, int sectorSize, int[] classCaps, bool bornSealed) + /// Bytes currently parked on this shard's owner-local chains; owner-only, no atomics. + internal long localBytes; + + /// Number of size classes with a non-empty owner-local chain; owner-only, no atomics. + internal int activeClasses; + + /// Ceiling for , shared across this shard's size classes. + internal readonly long localByteCap; + + internal ThreadShard(SectorAlignedBufferPool pool, int numClasses, int sectorSize, int[] classCaps, long localByteCap, bool bornSealed) { this.pool = pool; + this.localByteCap = localByteCap; state = bornSealed ? Sealed : Alive; buckets = new Bucket[numClasses]; for (var c = 0; c < numClasses; c++) @@ -162,9 +172,10 @@ internal ThreadShard(SectorAlignedBufferPool pool, int numClasses, int sectorSiz /// No per-push allocation. ConcurrentStack.Push allocates a link node per item; this pool /// exists to avoid exactly that. pushes into a pre-grown array. /// - /// Contention is a non-issue: the depot is already striped DepotStripes ways, is reached only on - /// magazine overflow/underflow, and each critical section is O(1). A lock-free CAS on one shared head would - /// reintroduce the cross-core ping-pong on a single cache line that this design removes. + /// Contention is bounded by striping: the depot is spread DepotStripes ways, sized from the + /// machine's processor count, is reached only on magazine overflow/underflow, and each critical section + /// is O(1). A lock-free CAS on one shared head would reintroduce the cross-core ping-pong on a single + /// cache line that this design removes. /// internal sealed class DepotStripe { @@ -226,10 +237,17 @@ public sealed partial class SectorAlignedBufferPool private const int NumClasses = LinearClasses + 2 * GeometricDoublings; // 28 (2 classes per doubling) private const int MaxPooledSectors = LinearTopSectors << GeometricDoublings; // 32768 (16 MB at 512 B sectors) - // Soft reuse targets (the byte budget is the only hard bound). - private const int LocalCap = 128; // buffers retained per (thread, class) before spilling to depot - private const long LocalByteCap = 32L << 20; // and a per-(thread, class) byte ceiling so large classes can't park the whole budget on one thread - private const int DepotStripes = 8; // power of two + // Local retention is bounded in bytes, per thread. Bytes are the resource the budget is denominated in; + // a buffer count is not, since one size class's buffer is up to 512x another's, and the count a thread + // needs is its in-flight IO pipeline depth, which is a property of the caller rather than of the pool. + // A thread's ceiling is its equal slice of the sub-budget that local caching draws from, so the slices + // of the expected concurrent threads sum to that sub-budget and no thread can retain enough to starve + // the others. Within a thread the ceiling is shared across size classes and enforced only once the + // thread reaches it: a thread whose traffic is a single class keeps the whole slice, and at the ceiling + // a class below its equal share reclaims from the class furthest above its share (see TryMakeRoom). + private const long MinThreadLocalBytes = 1L << 20; // floor, so a small configured budget still caches + private static readonly int DepotStripes = ConcurrencySharding.DepotStripeCount; // power of two + private static readonly int DepotStripeMask = DepotStripes - 1; private const int DepotStripeCap = 1024; // buffers per depot stripe private const int InitialRegistryCompactThreshold = 64; // compact dead shard weak-references once the registry first exceeds this @@ -268,6 +286,7 @@ public sealed partial class SectorAlignedBufferPool private BudgetState smallBudget; // isolated sub-budget for small classes (capacity <= LargeTierMinBytes) private BudgetState largeBudget; // isolated sub-budget for large (record/flush) classes private int firstLargeClass; // classes at or above this index draw from largeBudget + private long threadLocalByteCap; // per-thread ceiling on locally-retained (small-class) bytes private int[] classCaps; // capacity in sectors per class private DepotStripe[] depot; // [NumClasses * DepotStripes] private List> registry; @@ -285,6 +304,8 @@ private void InitOriginReturn() var smallBudgetBytes = ManagedBudgetBytes / SmallBudgetDivisor; smallBudget = new BudgetState(smallBudgetBytes); largeBudget = new BudgetState(ManagedBudgetBytes - smallBudgetBytes); + // Local caching parks only small classes, so a thread's slice is cut from the small sub-budget. + threadLocalByteCap = Math.Max(smallBudgetBytes / ConcurrencySharding.ExpectedConcurrentThreads, MinThreadLocalBytes); classCaps = new int[NumClasses]; firstLargeClass = NumClasses; for (var c = 0; c < NumClasses; c++) @@ -378,6 +399,9 @@ private unsafe SectorAlignedMemory GetOriginReturn(int required_bytes, int requi bucket.localHead = page.next; bucket.localCount--; bucket.localBytes -= page.permitBytes; + shard.localBytes -= page.permitBytes; + if (bucket.localCount == 0) + shard.activeClasses--; RecordReuse(cls); return PrepareForRent(page, bucket, required_bytes, clearOnReturn); } @@ -515,12 +539,11 @@ private void ReturnOriginReturn(SectorAlignedMemory page) return; } - // Large (record/flush) classes are shared globally via the striped depot instead of parking per-thread - // (on either the owner-local stack or the origin's cross-thread stack). With a wide value-size mix a - // thread rarely re-requests the same large class back-to-back, so per-thread parking strands big - // buffers and multiplies the working set across (thread x class), inflating RSS / GC. Routing both the - // owner and foreign returns of large buffers to the pool-owned depot lets any thread reuse them under - // the same byte budget, restoring legacy-like memory behavior; it also sidesteps the origin-shard + // Large (record/flush) classes are shared globally via the striped depot, on both the owner and the + // foreign return path. The per-thread tiers exploit a thread reusing the same size back-to-back; with + // a wide value-size mix a thread rarely re-requests the same large class, so a large buffer parked on + // its origin thread sits idle while the working set grows as (thread x class). One global working set + // under the same byte budget serves every thread, and keeps large returns clear of the origin-shard // finalize race (the depot is pool-owned). Large-buffer ops are low-rate and the workload is // bandwidth-bound, so the striped-depot handoff costs ~nothing. Small classes keep the atomic-free // per-thread origin-return fast path below. If the depot is closed (pool teardown), drop the buffer and @@ -586,35 +609,101 @@ private unsafe void FinalizeForReturn(SectorAlignedMemory page) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void PushLocal(Bucket bucket, SectorAlignedMemory page) { - if (!CanRetainLocal(bucket, page)) + var shard = bucket.owner; + var bytes = page.permitBytes; + if (shard.localBytes + bytes > shard.localByteCap && !TryMakeRoom(shard, bucket, bytes)) { if (!DepotPush(page.Level, page)) DropBuffer(page); return; } + RetainLocal(shard, bucket, page, bytes); + } + + /// Park a buffer on its bucket's owner-local chain and account for it on the owning shard. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void RetainLocal(ThreadShard shard, Bucket bucket, SectorAlignedMemory page, long bytes) + { + if (bucket.localCount == 0) + shard.activeClasses++; page.next = bucket.localHead; bucket.localHead = page; bucket.localCount++; - bucket.localBytes += page.permitBytes; + bucket.localBytes += bytes; + shard.localBytes += bytes; } - /// Whether a buffer may still be parked on the owner-local stack: bounded by both a count cap - /// and a byte ceiling (so a large-class buffer can't monopolize the pool's byte budget on one thread), - /// but always keeping at least one buffer for reuse locality. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool CanRetainLocal(Bucket bucket, SectorAlignedMemory page) - => bucket.localCount < LocalCap && (bucket.localCount == 0 || bucket.localBytes + page.permitBytes <= LocalByteCap); + /// + /// The thread is at its local byte ceiling. Admit for 's + /// class only if that class holds less than an equal share of the ceiling, making room by spilling the + /// class furthest above its share to the shared depot, where any thread can still reuse those buffers. + /// Reached only at the ceiling, so a thread whose traffic is a single size class keeps the whole slice; + /// under contention the outcome is max-min fair across the classes that thread is actually using. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private bool TryMakeRoom(ThreadShard shard, Bucket bucket, long bytes) + { + // The requester counts as active even when its chain is currently empty, so a starved class is not + // shut out by the classes crowding it. A thread using a single class therefore gets share == cap and + // fails here without scanning, which is the steady state once its chain is full. + var active = bucket.localCount == 0 ? shard.activeClasses + 1 : shard.activeClasses; + var share = shard.localByteCap / active; + if (bucket.localBytes + bytes > share) + return false; + + var buckets = shard.buckets; + Bucket victim = null; + while (shard.localBytes + bytes > shard.localByteCap) + { + if (victim is null || victim.localBytes <= share || victim.localHead is null) + victim = WorstOverShare(buckets, share); + if (victim is null) + return false; + SpillOneLocal(shard, victim); + } + return true; + } + + /// The owner-local chain holding the most bytes above , or null if none is. + private Bucket WorstOverShare(Bucket[] buckets, long share) + { + Bucket victim = null; + var worst = 0L; + for (var c = 0; c < firstLargeClass; c++) + { + var candidate = buckets[c]; + var over = candidate.localBytes - share; + if (over > worst && candidate.localHead is not null) + { + worst = over; + victim = candidate; + } + } + return victim; + } + + /// Move one buffer off an owner-local chain to the shared depot, dropping it if the depot is full. + private void SpillOneLocal(ThreadShard shard, Bucket victim) + { + var page = victim.localHead; + victim.localHead = page.next; + victim.localCount--; + victim.localBytes -= page.permitBytes; + shard.localBytes -= page.permitBytes; + if (victim.localCount == 0) + shard.activeClasses--; + if (!DepotPush(page.Level, page)) + DropBuffer(page); + } private void SpliceIntoLocal(Bucket bucket, SectorAlignedMemory rest) { + var shard = bucket.owner; var node = rest; - while (node is not null && CanRetainLocal(bucket, node)) + while (node is not null && shard.localBytes + node.permitBytes <= shard.localByteCap) { var nx = node.next; - node.next = bucket.localHead; - bucket.localHead = node; - bucket.localCount++; - bucket.localBytes += node.permitBytes; + RetainLocal(shard, bucket, node, node.permitBytes); node = nx; } while (node is not null) @@ -662,7 +751,7 @@ private static SectorAlignedMemory ClaimCrossThread(Bucket bucket) // ---- Depot --------------------------------------------------------------------------------------------- [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ThreadStripe() => Environment.CurrentManagedThreadId & (DepotStripes - 1); + private static int ThreadStripe() => Environment.CurrentManagedThreadId & DepotStripeMask; private bool DepotPush(int cls, SectorAlignedMemory page) { @@ -677,7 +766,7 @@ private SectorAlignedMemory DepotPop(int cls) var start = ThreadStripe(); for (var i = 0; i < DepotStripes; i++) { - var page = depot[baseIdx + ((start + i) & (DepotStripes - 1))].TryPop(); + var page = depot[baseIdx + ((start + i) & DepotStripeMask)].TryPop(); if (page is not null) return page; } @@ -757,7 +846,7 @@ private ThreadShard CreateShardSlow(int slot) lock (registryLock) { var bornSealed = poolState != PoolActive; - shard = new ThreadShard(this, NumClasses, sectorSize, classCaps, bornSealed); + shard = new ThreadShard(this, NumClasses, sectorSize, classCaps, threadLocalByteCap, bornSealed); if (bornSealed) shard.drainedOnce = 1; // nothing cached; no drain, no permits held else @@ -883,6 +972,8 @@ private void SealAndDrainShard(ThreadShard shard) ReleaseChainPermits(localChain); } } + shard.localBytes = 0; + shard.activeClasses = 0; } // Hand repair of any resurrected buffers back to ~ThreadShard by releasing drainedOnce: once this @@ -1018,6 +1109,26 @@ private static void RecordBypassAlloc() internal long LargeReservedBytes => largeBudget?.Used ?? 0; /// First size-class index that draws from the large sub-budget. Test-only. internal int FirstLargeClass => firstLargeClass; + /// Per-thread ceiling on locally-retained small-class bytes. Test-only. + internal long ThreadLocalByteCap => threadLocalByteCap; + /// Bytes the calling thread holds on all of its owner-local chains. Test-only. + internal long CallerLocalBytes => CallerShard()?.localBytes ?? 0; + /// Bytes the calling thread holds on its owner-local chain for . Test-only. + internal long CallerLocalBytesForClass(int cls) => CallerShard()?.buckets[cls].localBytes ?? 0; + + /// The calling thread's shard for this pool, or null if it has none. Test-only. + private ThreadShard CallerShard() + { + var arr = t_shards; + var slot = slotIndex; + if (arr is not null && slot < arr.Length) + { + var s = arr[slot]; + if (s is not null && ReferenceEquals(s.pool, this)) + return s; + } + return null; + } /// Total managed buffer allocations served by this pool (reuse-efficiency measure). Test-only. internal long TotalManagedAllocations => Interlocked.Read(ref totalManagedAllocations); /// Number of live shards registered with this pool. Test-only. diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/ConcurrencySharding.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/ConcurrencySharding.cs new file mode 100644 index 00000000000..9ff57894db4 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/ConcurrencySharding.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Numerics; + +namespace Tsavorite.core +{ + /// + /// Shared sizing for the structures that shard themselves across threads so that a single counter, + /// queue, or lock does not become a contended cache line once dozens of threads touch it on every + /// operation: the device in-flight tracking (NativeStorageDevice.NumShards) and the buffer + /// pool's shared overflow depot. + /// + /// The 2 × ProcessorCount term scales the count to the machine + /// ( honors process CPU affinity and cgroup limits); the cap + /// bounds the memory each shard costs and the length of whatever scan walks the shards. Internal + /// implementation details, not user knobs. + /// + /// + internal static class ConcurrencySharding + { + /// Formula: two shards per logical processor, capped. + internal static int Compute(int cap) => Math.Min(2 * Environment.ProcessorCount, cap); + + /// + /// Same formula, clamped to and rounded up to a power of two, for callers + /// that index shards with a mask rather than a modulo. + /// + internal static int ComputePow2(int min, int cap) + => (int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(Compute(cap), min)); + + /// + /// Device in-flight shard count. Cap 32 bounds the slot table and the O(shards) in-flight scan; + /// submitters past that share shards, which the per-thread throttle keeps safe. + /// + internal static readonly int NumShardCount = Compute(32); + + /// + /// Buffer pool depot stripe count, a power of two so the depot can be indexed with a mask. Each + /// stripe is an independently locked stack, so this is how many threads can touch the shared depot + /// concurrently without serializing. Floor 8 keeps striping on low-processor boxes; cap 64 bounds + /// the per-class stripe array (~72 bytes per stripe per size class) and the scan a depot miss + /// performs. + /// + internal static readonly int DepotStripeCount = ComputePow2(min: 8, cap: 64); + + /// + /// Expected number of threads concurrently renting from one buffer pool. Used to divide the pool's + /// cacheable byte budget into equal per-thread slices, so that the slices of that many threads sum to + /// the budget and no thread can retain enough to starve the others. + /// + internal static readonly int ExpectedConcurrentThreads = Compute(64); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SectorAlignedBufferPoolTests.cs b/libs/storage/Tsavorite/cs/test/SectorAlignedBufferPoolTests.cs index 8c93ff113aa..971bb760089 100644 --- a/libs/storage/Tsavorite/cs/test/SectorAlignedBufferPoolTests.cs +++ b/libs/storage/Tsavorite/cs/test/SectorAlignedBufferPoolTests.cs @@ -438,6 +438,96 @@ public void SmallBudgetIsolatedFromLargeExhaustion() ClassicAssert.AreEqual(0, pool.ReservedBytes, "budget must return to zero after Free"); } + // ---- Per-thread local retention ------------------------------------------------------------------------ + + /// Rent buffers of simultaneously, then return them all. + private static void RentBurstAndReturn(SectorAlignedBufferPool pool, int size, int count) + { + var held = new List(count); + for (var i = 0; i < count; i++) + held.Add(pool.Get(size, clearOnReturn: false)); + foreach (var p in held) + p.Return(); + } + + [Test] + public void LocalRetentionIsBoundedByThePerThreadByteCap() + { + // A burst far larger than the thread's slice must spill to the shared depot rather than park on the + // thread: local retention is bounded in bytes, not by a per-class buffer count. + SectorAlignedBufferPool.ManagedBudgetBytes = 64L << 20; + var pool = new SectorAlignedBufferPool(1, SectorSize); + try + { + var cap = pool.ThreadLocalByteCap; + ClassicAssert.Greater(cap, 0, "pool must publish a per-thread local byte cap"); + + RentBurstAndReturn(pool, SectorSize, (int)(cap / (2 * SectorSize)) * 3); + + ClassicAssert.Greater(pool.CallerLocalBytes, 0, "the thread must retain buffers up to its slice"); + ClassicAssert.LessOrEqual(pool.CallerLocalBytes, cap, "a thread must not retain more than its slice"); + } + finally { pool.Free(); } + ClassicAssert.AreEqual(0, pool.ReservedBytes, "budget must return to zero after Free"); + } + + [Test] + public void LocalByteCapIsSharedFairlyAcrossClasses() + { + // The slice is shared across the classes a thread uses: one class alone may hold all of it, but a + // second active class reclaims down to an equal share instead of being shut out. + SectorAlignedBufferPool.ManagedBudgetBytes = 64L << 20; + var pool = new SectorAlignedBufferPool(1, SectorSize); + try + { + var cap = pool.ThreadLocalByteCap; + var clsA = SectorAlignedBufferPool.TestClassOfSectors(1); // SectorSize + var clsB = SectorAlignedBufferPool.TestClassOfSectors(8); // 4 KB + ClassicAssert.AreNotEqual(clsA, clsB, "the two probe sizes must land in different classes"); + + // Only class A is active: it may take the whole slice. + RentBurstAndReturn(pool, SectorSize, (int)(cap / (2 * SectorSize)) + 64); + ClassicAssert.Greater(pool.CallerLocalBytesForClass(clsA), cap / 2, + "a thread using a single size class must be able to hold the whole slice"); + + // Class B becomes active against a full slice: it must reclaim toward an equal share. + RentBurstAndReturn(pool, 4096, (int)(cap / (2 * 9 * SectorSize)) + 64); + + ClassicAssert.LessOrEqual(pool.CallerLocalBytes, cap, "the shared slice must still bound the total"); + ClassicAssert.Greater(pool.CallerLocalBytesForClass(clsB), 0, + "a newly active class must obtain a share of the slice"); + ClassicAssert.LessOrEqual(pool.CallerLocalBytesForClass(clsA), cap * 3 / 4, + "the incumbent class must give bytes back toward its share"); + } + finally { pool.Free(); } + ClassicAssert.AreEqual(0, pool.ReservedBytes, "budget must return to zero after Free"); + } + + [Test] + public void MixedClassBurstStaysWithinOneThreadSlice() + { + // A thread cycling through many size classes must hold no more than its slice in total. A per-class + // count cap would admit a full chain per class, so the thread's total would scale with the number of + // classes it touches rather than being bounded. + SectorAlignedBufferPool.ManagedBudgetBytes = 64L << 20; + var pool = new SectorAlignedBufferPool(1, SectorSize); + try + { + var cap = pool.ThreadLocalByteCap; + for (var round = 0; round < 4; round++) + { + for (var sectors = 1; sectors <= 16; sectors++) + RentBurstAndReturn(pool, sectors * SectorSize, 64); + } + + ClassicAssert.LessOrEqual(pool.CallerLocalBytes, cap, + "a thread's total local retention must be bounded across all size classes it uses"); + ClassicAssert.Greater(pool.CallerLocalBytes, 0, "the thread must still cache within its slice"); + } + finally { pool.Free(); } + ClassicAssert.AreEqual(0, pool.ReservedBytes, "budget must return to zero after Free"); + } + // ---- Dead-thread permit reclamation -------------------------------------------------------------------- [Test] diff --git a/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs index c3866c9ca4a..ec8f33df529 100644 --- a/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs @@ -995,7 +995,7 @@ public unsafe void IDevice_PermissionDeniedAtFirstWrite_CallbackGetsError(Device // - multiple completion threads / io_contexts (pick_context / pick_ring sharding); // - multiple devices interleaved across threads (the pick_context/pick_ring owner+bounds // thread-local guard, whose absence is an out-of-bounds shard index); - // - ThrottleLimit -> submission-ring depth sizing, including the clamp above MaxThrottle; + // - ThrottleLimit -> aggregate in-flight cap (capped at io-contexts * queue-depth); // - the late P/Invoke entry points GetFileSize / RemoveSegment / Reset / TryComplete. // // io_uring cases self-skip when the loaded native library / kernel lacks a working io_uring. @@ -1037,10 +1037,15 @@ static unsafe bool ProbeUringWorks() /// /// Build + Initialize a Native device on the requested backend, completion-thread count and - /// throttle limit. Self-skips on non-Linux, or when io_uring is requested but unavailable. + /// throttle limit. (kernel io_context / io_uring ring count, + /// decoupled from the drainer count), (per-ring submission depth) + /// and (io_uring SQPOLL) exercise the corresponding ctor knobs; + /// each 0/false selects the device default. Self-skips on non-Linux, or when io_uring is + /// requested but unavailable. /// static NativeStorageDevice CreateNativeForTest(string path, long segmentSize, NativeBackend backend, - int completionThreads = 1, int throttleLimit = 0, bool omitSegmentId = false) + int completionThreads = 1, int throttleLimit = 0, bool omitSegmentId = false, + int numIoContexts = 0, int queueDepth = 0, bool uringSqPoll = false) { if (!OperatingSystem.IsLinux()) Assert.Ignore("NativeStorageDevice is Linux-only."); @@ -1053,7 +1058,8 @@ static NativeStorageDevice CreateNativeForTest(string path, long segmentSize, Na NativeBackend.Uring => NativeStorageDevice.IoBackend.Uring, _ => NativeStorageDevice.IoBackend.Default, }; - var d = new NativeStorageDevice(path, deleteOnClose: true, numCompletionThreads: completionThreads, ioBackend: io); + var d = new NativeStorageDevice(path, deleteOnClose: true, numCompletionThreads: completionThreads, ioBackend: io, + numIoContexts: numIoContexts, queueDepth: queueDepth, uringSqPoll: uringSqPoll); if (throttleLimit > 0) d.ThrottleLimit = throttleLimit; d.Initialize(segmentSize, omitSegmentIdFromFilename: omitSegmentId && segmentSize == -1L); @@ -1249,7 +1255,7 @@ public unsafe void Native_HighThrottle_HighInFlight(NativeBackend backend) [Category("IDevice")] public unsafe void Native_ThrottleAboveMax_IsClampedAndStillWorks(NativeBackend backend) { - // A throttle far above the kernel-safe ceiling (MaxThrottle=4096) is clamped (and warned); + // A throttle far above the kernel capacity (io-contexts * queue-depth) is capped (and warned); // the device must still round-trip correctly. using var device = CreateNativeForTest(Path.Join(TestUtils.MethodTestDir, "test.log"), 64 * Mib, backend, throttleLimit: 100_000); const int size = 4 * 1024; @@ -1330,6 +1336,141 @@ public unsafe void Native_HighConcurrency_ManyThreads_NoHang(NativeBackend backe GC.KeepAlive(wbuf); GC.KeepAlive(rroots); } + // ----- io_context / ring-count / queue-depth / SQPOLL ctor knobs -------------------------- + + /// + /// Issues one read per buffer, striped across background threads, + /// and returns once every read has been submitted (completions are awaited by the caller). + /// Native rings/io_contexts are picked per submitting thread, so a single-threaded issue loop puts + /// every read on one ring; fanning out is what actually exercises a multi-ring configuration. + /// + void SubmitReadsFromThreads(IDevice device, IntPtr[] rptrs, int size, int submitThreads) + { + var submitters = new Thread[submitThreads]; + for (int t = 0; t < submitThreads; t++) + { + int start = t; + submitters[t] = new Thread(() => + { + for (int i = start; i < rptrs.Length; i += submitThreads) + device.ReadAsync(0, (ulong)((long)i * size), rptrs[i], (uint)size, IOCallback, null); + }) + { IsBackground = true }; + submitters[t].Start(); + } + foreach (var s in submitters) s.Join(); + } + + [Test] + [TestCase(NativeBackend.Libaio)] + [TestCase(NativeBackend.Uring)] + [Category("IDevice")] + public unsafe void Native_ExplicitIoContexts_DefaultDepth_MultiRing(NativeBackend backend) + { + // Decouple ring count from drainer count: 8 io_contexts served by 2 drainers, default + // (auto-derived) queue depth. For libaio + default queueDepth this also exercises the + // ResolveLibaioReservationDepth branch (the reservation is sized from the throttle share + // rather than the io_uring per-ring ceiling). Rings are assigned per submitting thread + // (thread-affine), so the reads are issued from 8 concurrent threads to actually fan out + // across all 8 rings rather than piling onto the single test thread's ring. + const int N = 128, size = 4 * 1024, submitThreads = 8; + using var device = CreateNativeForTest(Path.Join(TestUtils.MethodTestDir, "test.log"), 256 * Mib, backend, + completionThreads: 2, throttleLimit: 1024, numIoContexts: 8); + + var (wbuf, wptr) = AllocateAlignedBuffer(N * size, j => (byte)(((j / size) * 5 + (j % size)) & 0xFF)); + device.WriteAsync(wptr, 0, 0, (uint)(N * size), IOCallback, null); + semaphore.Wait(); + + var rbufs = new byte[N][]; + var rptrs = new IntPtr[N]; + for (int i = 0; i < N; i++) + { + var (rb, rp) = AllocateAlignedBuffer(size, _ => 0); + rbufs[i] = rb; rptrs[i] = rp; + } + + SubmitReadsFromThreads(device, rptrs, size, submitThreads); + for (int i = 0; i < N; i++) semaphore.Wait(); + + for (int i = 0; i < N; i++) + { + int blk = i; + AssertBufferContents(rptrs[i], size, off => (byte)((blk * 5 + off) & 0xFF), $"{backend} block {blk}"); + } + GC.KeepAlive(wbuf); GC.KeepAlive(rbufs); + } + + [Test] + [TestCase(NativeBackend.Libaio)] + [TestCase(NativeBackend.Uring)] + [Category("IDevice")] + public unsafe void Native_ExplicitQueueDepth_RoundTrips(NativeBackend backend) + { + // Explicit shallow per-ring queue depth (64) across 4 rings, and fire more concurrent reads + // (256) than the aggregate ring capacity so the native ring-full backpressure (submit unwinds + // to Pending and retries after a completion) is exercised and every read still completes. + // Reads are issued from 4 concurrent threads so all 4 shallow rings are driven at once. + const int N = 256, size = 4 * 1024, submitThreads = 4; + using var device = CreateNativeForTest(Path.Join(TestUtils.MethodTestDir, "test.log"), 256 * Mib, backend, + completionThreads: 2, throttleLimit: 256, numIoContexts: 4, queueDepth: 64); + + var (wbuf, wptr) = AllocateAlignedBuffer(N * size, j => (byte)(((j / size) * 3 + (j % size)) & 0xFF)); + device.WriteAsync(wptr, 0, 0, (uint)(N * size), IOCallback, null); + semaphore.Wait(); + + var rbufs = new byte[N][]; + var rptrs = new IntPtr[N]; + for (int i = 0; i < N; i++) + { + var (rb, rp) = AllocateAlignedBuffer(size, _ => 0); + rbufs[i] = rb; rptrs[i] = rp; + } + + SubmitReadsFromThreads(device, rptrs, size, submitThreads); + for (int i = 0; i < N; i++) semaphore.Wait(); + + for (int i = 0; i < N; i++) + { + int blk = i; + AssertBufferContents(rptrs[i], size, off => (byte)((blk * 3 + off) & 0xFF), $"{backend} block {blk}"); + } + GC.KeepAlive(wbuf); GC.KeepAlive(rbufs); + } + + [Test] + [Category("IDevice")] + public unsafe void Native_Uring_SqPoll_RoundTrips() + { + // io_uring SQPOLL is Uring-only (libaio/Windows ignore it). Each ring gets its own kernel + // poll thread, so the reads are issued from 4 concurrent threads to drive all 4 rings (and + // their poll threads) through the SQPOLL submit branch. + const int N = 32, size = 4 * 1024, submitThreads = 4; + using var device = CreateNativeForTest(Path.Join(TestUtils.MethodTestDir, "test.log"), 128 * Mib, NativeBackend.Uring, + completionThreads: 2, throttleLimit: 512, numIoContexts: 4, uringSqPoll: true); + + var (wbuf, wptr) = AllocateAlignedBuffer(N * size, j => (byte)(((j / size) * 9 + (j % size)) & 0xFF)); + device.WriteAsync(wptr, 0, 0, (uint)(N * size), IOCallback, null); + semaphore.Wait(); + + var rbufs = new byte[N][]; + var rptrs = new IntPtr[N]; + for (int i = 0; i < N; i++) + { + var (rb, rp) = AllocateAlignedBuffer(size, _ => 0); + rbufs[i] = rb; rptrs[i] = rp; + } + + SubmitReadsFromThreads(device, rptrs, size, submitThreads); + for (int i = 0; i < N; i++) semaphore.Wait(); + + for (int i = 0; i < N; i++) + { + int blk = i; + AssertBufferContents(rptrs[i], size, off => (byte)((blk * 9 + off) & 0xFF), $"Uring-SQPOLL block {blk}"); + } + GC.KeepAlive(wbuf); GC.KeepAlive(rbufs); + } + // ----- late P/Invoke entry points (GetFileSize / Reset / TryComplete / RemoveSegment) ----- [Test] diff --git a/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs b/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs index a45966c5b6c..d00d545f606 100644 --- a/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs @@ -1559,6 +1559,15 @@ static async Task UpgradeReplicasAsync(ClusterTestContext context, IPEndPoint re break; } + + // The role flips to "master" (via TryTakeOverForPrimary) before the failover + // session clears its recovery flag (EndRecovery) and reports completion. While a + // promoted primary is still recovering, reads to its own slots are answered with + // "CLUSTERDOWN Hash slot not served" (see ClusterSlotVerify). Wait for the failover + // to fully complete on both nodes so the immediately-following reads don't race + // that window. + context.clusterTestUtils.WaitForFailoverCompleted(replica1, context.logger); + context.clusterTestUtils.WaitForFailoverCompleted(replica2, context.logger); } } } diff --git a/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs b/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs index 02a5a6e0a62..0676e5cecb9 100644 --- a/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs +++ b/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs @@ -3404,10 +3404,13 @@ public void WaitForNoFailover(int nodeIndex, ILogger logger = null) } public void WaitForFailoverCompleted(int nodeIndex, ILogger logger = null) + => WaitForFailoverCompleted((IPEndPoint)endpoints[nodeIndex], logger); + + public void WaitForFailoverCompleted(IPEndPoint endPoint, ILogger logger = null) { while (true) { - var infoItem = context.clusterTestUtils.GetReplicationInfo(nodeIndex, [ReplicationInfoItem.LAST_FAILOVER_STATE], logger: context.logger); + var infoItem = GetReplicationInfo(endPoint, [ReplicationInfoItem.LAST_FAILOVER_STATE], logger: logger); if (infoItem[0].Item2.Equals("failover-completed")) break; BackOff(cancellationToken: context.cts.Token, msg: nameof(WaitForFailoverCompleted)); diff --git a/test/standalone/Garnet.test/GarnetServerConfigTests.cs b/test/standalone/Garnet.test/GarnetServerConfigTests.cs index f546a8e5bd1..67d33446369 100644 --- a/test/standalone/Garnet.test/GarnetServerConfigTests.cs +++ b/test/standalone/Garnet.test/GarnetServerConfigTests.cs @@ -233,6 +233,107 @@ public void LoadModuleCsInvalidSpecIsRejected() ClassicAssert.AreEqual(0, invalidOptions.Count); } + [Test] + public void DeviceIoContextsAndQueueDepthOptions() + { + // Defaults: with no explicit override the values come from defaults.conf (0 = device default) + // and flow through to GarnetServerOptions unchanged. + { + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments([], out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, options.DeviceIoContexts); + ClassicAssert.AreEqual(0, options.DeviceQueueDepth); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(0, serverOptions.DeviceIoContexts); + ClassicAssert.AreEqual(0, serverOptions.DeviceQueueDepth); + } + + // Explicit values are parsed and flow through to GarnetServerOptions. + { + var args = new[] { "--device-io-contexts", "96", "--device-queue-depth", "4096" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(96, options.DeviceIoContexts); + ClassicAssert.AreEqual(4096, options.DeviceQueueDepth); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(96, serverOptions.DeviceIoContexts); + ClassicAssert.AreEqual(4096, serverOptions.DeviceQueueDepth); + } + } + + [Test] + public void DeviceAioMaxDevicesOption() + { + var savedAioMaxDevices = NativeStorageDevice.AioMaxDevices; + try + { + // Default: with no explicit override the value comes from defaults.conf (32) and flows + // through to GarnetServerOptions unchanged; Initialize applies it to the process-global static. + { + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments([], out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(32, options.DeviceAioMaxDevices); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(32, serverOptions.DeviceAioMaxDevices); + + NativeStorageDevice.AioMaxDevices = 7; // perturb to prove Initialize reapplies + serverOptions.Initialize(); + ClassicAssert.AreEqual(32, NativeStorageDevice.AioMaxDevices); + } + + // Explicit value is parsed, flows through to GarnetServerOptions, and Initialize applies it. + { + var args = new[] { "--device-aio-max-devices", "64" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(64, options.DeviceAioMaxDevices); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(64, serverOptions.DeviceAioMaxDevices); + + serverOptions.Initialize(); + ClassicAssert.AreEqual(64, NativeStorageDevice.AioMaxDevices); + } + } + finally + { + NativeStorageDevice.AioMaxDevices = savedAioMaxDevices; + } + } + + [Test] + public void DeviceUringSqPollOptions() + { + // Defaults: opt-in SQPOLL is off and the idle window is the native default (0) per + // defaults.conf, flowing through to GarnetServerOptions unchanged. + { + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments([], out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(false, options.DeviceUringSqPoll); + ClassicAssert.AreEqual(0, options.DeviceUringSqPollIdleMs); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.IsFalse(serverOptions.DeviceUringSqPoll); + ClassicAssert.AreEqual(0, serverOptions.DeviceUringSqPollIdleMs); + } + + // Explicit values are parsed and flow through to GarnetServerOptions. + { + var args = new[] { "--device-uring-sqpoll", "true", "--device-uring-sqpoll-idle-ms", "2000" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(true, options.DeviceUringSqPoll); + ClassicAssert.AreEqual(2000, options.DeviceUringSqPollIdleMs); + + var serverOptions = options.GetServerOptions(); + ClassicAssert.IsTrue(serverOptions.DeviceUringSqPoll); + ClassicAssert.AreEqual(2000, serverOptions.DeviceUringSqPollIdleMs); + } + } + [Test] public void ImportExportConfigLocal() { diff --git a/website/docs/dev/device-tuning.md b/website/docs/dev/device-tuning.md new file mode 100644 index 00000000000..88f3d07ca95 --- /dev/null +++ b/website/docs/dev/device-tuning.md @@ -0,0 +1,376 @@ +--- +id: device-tuning +sidebar_label: Device Tuning +title: Native Storage Device Tuning +--- + +This page documents the tuning surface of Garnet's **Native storage device** — the +Linux `io_uring` / `libaio` `IDevice` implementation used when the hybrid log is tiered to +an NVMe/SSD (`--storage-tier`). It covers every `--device-*` knob, its default, the exact +formulas the device uses to derive its internal parameters, the internal constants that +bound those formulas, and the definitions of **headroom**, **floor**, **cap**, and +**ceiling** as the code uses them. + +The Native device is the default on x64 Linux and is what lets a disk-served workload +approach raw-device (`fio`) IOPS. On Windows it uses the IOCP thread-pool backend and most +Linux-only knobs below are ignored. + +## Mental model: three orthogonal dimensions + software backpressure + +A kernel async-IO device has exactly two physical capacity dimensions plus one software +policy. Garnet exposes each as its own knob so that no single number silently controls two +things: + +| Dimension | Knob | What it sizes | Analogy (fio) | +|---|---|---|---| +| **Ring count** `N` | `--device-io-contexts` | number of independent kernel submission queues (io_uring rings / libaio `io_context`s) | `numjobs` / parallel submission queues | +| **Ring depth** `D` | `--device-queue-depth` | per-ring kernel submission depth (`maxEvents` passed to `io_uring_queue_init` / `io_setup`) | per-queue `iodepth` | +| **Aggregate in-flight** `T` | `--device-throttle-limit` | max reads the allocator keeps in flight before applying backpressure | total effective `iodepth` | + +The only relationship between the three is a single correctness invariant: + +``` +aggregate in-flight T ≤ kernel capacity N × D +``` + +`T` is **software backpressure** enforced by the Tsavorite allocator — it bounds how much +pending-read work (and thus how much pinned read-buffer memory) accumulates. `N` and `D` +size actual kernel structures. Keeping them as separate knobs lets ring count and ring +depth be sized independently. + +## The tuning knobs + +All knobs have a command-line form (`--device-…`), a config-file form (the PascalCase name), +and a default. Defaults live in `libs/host/defaults.conf`; the option definitions in +`libs/host/Configuration/Options.cs`. + +| CLI flag | Config key | Default | Applies to | Meaning | +|---|---|---|---|---| +| `--device-type` | `DeviceType` | `Default` | all | `Default` (Native on x64 Linux/Windows, else RandomAccess), `Native`, `RandomAccess`, `FileStream`, `AzureStorage`, `Null`. | +| `--device-io-backend` | `DeviceIoBackend` | `Default` | Native, Linux | `Default` (= `Libaio`), `Libaio`, or `Uring` (io_uring). | +| `--device-completion-threads` | `DeviceCompletionThreads` | `4` (max 64) | Native, Linux | Number of background IO **completion drain** threads. | +| `--device-io-contexts` | `DeviceIoContexts` | `0` (→ smart default) | Native, Linux | Ring **count** `N` (see [smart default](#derived-smart-io-context-default)). | +| `--device-queue-depth` | `DeviceQueueDepth` | `0` (→ 4096) | Native, Linux | Per-ring **depth** `D`. | +| `--device-throttle-limit` | `DeviceThrottleLimit` | `0` (→ 4096 Native / 120 managed) | all devices | Aggregate **in-flight** `T` (`IDevice.ThrottleLimit`). | +| `--device-aio-max-devices` | `DeviceAioMaxDevices` | `32` | Native, Linux, **libaio only** | Target number of libaio devices to fit within the machine-global `fs.aio-max-nr` budget. | + +### `--device-io-backend` + +Selects the Linux kernel async-IO API for the Native device. + +* **`Libaio`** (also `Default`) — classic Linux AIO (`io_submit`/`io_getevents`). Each ring is + a kernel `io_context`. libaio is **ring-count-neutral**: its kernel `io_context` mutex is + cheap, so a handful of rings already saturates it. **Caveat:** `io_setup` permanently + reserves `N × D` events from the *machine-global* `fs.aio-max-nr` budget (see + [libaio reservation](#derived-libaio-reservation-depth)). +* **`Uring`** — io_uring. Each ring is an `io_uring` instance with its own SQ/CQ mmap memory + (no global budget). io_uring is **ring-count-sensitive**: when rings `<` submitter + concurrency, submitters serialize on a per-ring submit lock and throughput can drop ~3×. + Always set `--device-io-contexts` at or above your concurrency (the [smart default](#derived-smart-io-context-default) + does this for you). + +The shipped `libnative_device.so` is built with `-DUSE_URING=ON`, so `liburing.so.2` must be +present at load time for **all** backends; a `-DUSE_URING=OFF` rebuild only needs `libaio`. + +### `--device-completion-threads` + +Number of background threads that drain completions from the rings. Under high concurrent +pending-read load a single drainer convoys on the completion-signal path and collapses +throughput; the default of `4` removes that on both backends. io_uring scales further and +more CPU-efficiently with more drainers; libaio benefits little beyond a few. Drainers +range-drain contiguous slices of the rings, so `N` is always clamped up to at least the +drainer count (every drainer owns ≥ 1 ring). + +Inline draining (a submitter thread reaping its own ring's completions before waiting) is +always on and is the primary completion mechanism at the serving peak; the background +drainers are a backstop for rings whose owning thread is idle. + +### `--device-io-contexts` (ring count `N`) + +The single most important knob for **io_uring**. It is the ring count, decoupled from the +drainer count. Set it at or above your submitter concurrency (roughly your connection count) +so each submitter owns a ring and `io_submit` is contention-free. Too few rings serialize +submitters on the per-ring lock (~3× slower). **libaio is +largely indifferent** to it. `0` selects the [smart default](#derived-smart-io-context-default). + +### `--device-queue-depth` (ring depth `D`) + +Per-ring kernel submission depth. Orthogonal to `N` and `T`. `0` selects the default of +`4096`, [capped](#floor-cap-ceiling-headroom) at the io_uring hard limit of `32768`. For +libaio the effective `io_setup` reservation is sized down from this default (see +[below](#derived-libaio-reservation-depth)); set it explicitly to reserve a specific depth. + +### `--device-throttle-limit` (aggregate in-flight `T`) + +`IDevice.ThrottleLimit` — the max reads kept in flight before the allocator applies +backpressure and drains instead of issuing more. `0` uses the device's built-in default: +**4096** for the Native device (deep NVMe / io_uring queues), **120** for the managed in-box +devices. It is a pure software / memory limit (its footprint is `T ×` sector-aligned read +size of pinned read buffers); in the split model it sizes nothing in the kernel. It is +[capped](#derived-effective-throttle) at the kernel capacity `N × D`. + +### `--device-aio-max-devices` (libaio budget divisor) + +libaio only. `fs.aio-max-nr` is a *machine-global* event budget shared by every device in +every process. `io_setup` permanently draws `N × D` events from it per device. This knob is +the **target number of Native libaio devices to fit** within that budget: the default per-device +reservation is hard-capped at `fs.aio-max-nr / this` (default 32), keeping at least this many +devices creatable regardless of `--device-completion-threads` / `--device-throttle-limit`. The +cap is best-effort — see [the reservation derivation](#derived-libaio-reservation-depth) for the +two cases it cannot cover. Raise `fs.aio-max-nr` (e.g. `sysctl -w fs.aio-max-nr=1048576`) or +lower this value to give each serving device a deeper reservation. Ignored for io_uring (no +global budget) and non-Linux. + +### `--device-uring-sqpoll` (io_uring submission polling) + +io_uring only. Enables `IORING_SETUP_SQPOLL` so a **kernel thread** polls the submission +queue and user-side submissions become syscall-free (no `io_uring_enter` per submit). Each +ring gets its **own** poll thread (no `IORING_SETUP_ATTACH_WQ`), so submission stays parallel +across rings; one poll thread shared across rings would serialize it. +`--device-uring-sqpoll-idle-ms` sets `sq_thread_idle` (how long a poll thread spins after the +last submit before parking; `0` = 10s native default). **Off by default (opt-in).** Ignored +for libaio / on Windows. + +With one poll thread per ring, SQPOLL **matches or slightly beats** the default per-submit +path on the 8×NVMe RAID-0 target (uring, 512B random reads), peaking at fio parity: + +| config (io-contexts / threads) | SQPOLL off | SQPOLL on | +|--------------------------------|-----------:|----------:| +| 8 / 16 | 3.10M | **3.44M**| +| 16 / 32 | 5.18M | **5.83M**| +| 32 / 32 (peak) | 8.12M | **8.39M**| +| 32 / 64 | 6.98M | **7.18M**| + +:::tip Let the kernel place the poll threads +The poll threads are left unpinned so the scheduler can spread them across node 0's mostly-idle +cores; pinning them onto the submitter / RESP cores costs throughput. They are busy-polling +kernel threads and consume CPU, so give them cores to run on; on core-starved hosts SQPOLL can +lose to the default path. +::: + +## Derived parameters + +The device computes several internal parameters at creation (first IO) from the knobs above +and the host environment. These are **not** directly settable; understanding them is the key +to tuning. + +### Derived: smart io-context default {#derived-smart-io-context-default} + +When `--device-io-contexts` is left at `0`: + +``` +uring : N = max(completion-threads, min(2 × ProcessorCount, 64)) +libaio: N = completion-threads +``` + +The value is then clamped up to `completion-threads` so every drainer owns at least one ring. +io_uring is ring-starved below submitter concurrency, so it defaults to a hardware-aware ring +count (`2 × cores`, **capped** at 64 to bound ring memory at ~400 KB/ring → ≤ ~25 MB). libaio +is ring-count-neutral and its `N × D` draws from the global budget, so it keeps the +conservative `rings = drainers` default. + +### Derived: queue depth `D` {#derived-queue-depth} + +``` +D = (--device-queue-depth > 0) ? --device-queue-depth : 4096 // DefaultQueueDepth +D = min(D, 32768) // MaxQueueDepth ceiling (io_uring hard limit) +``` + +`D = 4096` (a **ceiling**, not pre-allocated work) satisfies "per-ring depth ≥ per-ring +in-flight" for any `N` as long as `T ≤ 4096 × N` (the default `T = 4096` always fits), so no +ring ever stalls full. Deeper-than-needed rings are harmless; the only cost is bounded pinned +ring memory. + +### Derived: libaio reservation depth {#derived-libaio-reservation-depth} + +For **libaio with the default queue depth**, `D = 4096` would over-reserve from the global +`fs.aio-max-nr` budget (a libaio ring never actually holds more than the aggregate throttle +spread across the rings). So the reservation is sized to that throttle share instead +(`ResolveLibaioReservationDepth`): + +``` +share = ceil(T / N) // this ring's share of aggregate in-flight +depth = NextPow2(share × Headroom) // Headroom = 2 (over-provision factor) +depth = max(depth, 128) // Floor = LibaioReservationFloor +depth = min(depth, 2048) // Cap = LibaioReservationCap +depth = min(depth, D) // Ceiling = the resolved queue depth + +// Then a HARD per-device AIO-budget ceiling, independent of N and T: +perDeviceBudget = fs.aio-max-nr / --device-aio-max-devices // default fs.aio-max-nr / 32 +while (depth > 1 && N × depth > perDeviceBudget) depth >>= 1 // halve (stay pow2) until it fits +``` + +The caller then caps `effectiveThrottleLimit` at `N × depth` so aggregate in-flight tracks the +(possibly reduced) reservation. Consequences: + +* **Multi-ring serving devices** (`N ≥ 4`) keep `N × depth ≥ T` through the share clamps, so + those clamps drop the per-device global-budget footprint at no IOPS cost. The budget ceiling + below runs after them and can still reduce it. +* **Low-ring-count auxiliary devices** (e.g. cluster AOF / checkpoint logs created with the + raw `Devices.CreateLogDevice` single-ring defaults) drop to `≈ N × cap`, so the + `--device-aio-max-devices` target of them coexists within a stock 65536 budget. +* The hard per-device ceiling keeps at least `--device-aio-max-devices` devices fitting the + budget: on a stock 65536 budget it bounds each device to 2048 events; a host that sizes + `fs.aio-max-nr` for its workload keeps serving devices at full depth (e.g. `4194304 / 32 = + 131072` per device, which never binds). It is best-effort in two respects: `depth` cannot fall + below one event per ring, so an `N` above the per-device share still exceeds it (warned at + creation); and the budget is the machine total, not what remains after other processes. +* **When the budget ceiling binds it lowers the throttle**, overriding the share math above, because + `effectiveThrottleLimit` is capped at `N × depth ≤ perDeviceBudget`. On a stock 65536 budget + that bound is 2048, so the default `T = 4096` is halved at every ring count — worth ~9% on a + libaio disk-serving workload. Size `fs.aio-max-nr` for the host (`sysctl -w + fs.aio-max-nr=…`, persisted under `/etc/sysctl.d/`) so `fs.aio-max-nr / + --device-aio-max-devices ≥ T`; a serving host wanting the default `T = 4096` across 32 + devices needs `fs.aio-max-nr ≥ 131072`. + +io_uring skips all of this — it uses `D` directly (per-ring mmap memory, no global budget). + +### Derived: effective throttle {#derived-effective-throttle} + +``` +requestedThrottle = (ThrottleLimit > 0) ? ThrottleLimit : 4096 // DefaultThrottleLimit +kernelCapacity = N × D +effectiveThrottle = min(requestedThrottle, kernelCapacity) // cap at kernel capacity (warns if it binds) +``` + +Capping `T` at `N × D` enforces the "in-flight ≤ kernel capacity" invariant that prevents the +ring-full submit spin. This cap is **decoupled** from the depth cap, so a high-connection +deployment can raise `T` as long as `N × D` is large enough. The per-shard clamp described +next imposes a second, independent ceiling: raising `T` above +`NumShards × MaxPerThreadInFlight` (**4096** on any host with ≥ 16 logical processors) has no +effect, because the per-thread budget saturates at `MaxPerThreadInFlight`. + +### Derived: per-thread in-flight (sharding) {#derived-sharding} + +In-flight is tracked **per submitter thread** (sharded) rather than as one global counter, to +avoid cache-line contention at high IOPS. Each submitter is assigned a shard round-robin +(`AssignShard`, reduced modulo `NumShards` as `uint` so a long-running thread-churning server +never wraps to a negative index), and throttles on its own shard: + +``` +global = effectiveThrottle +active = activeShards // occupied shards (exact live count) +perThread = clamp(global / active, 1, 128) // 128 = MaxPerThreadInFlight +Throttle() = (this shard's in-flight) > perThread +``` + +`activeShards` is the number of currently *occupied* shards (shards with at least one in-flight +IO). It is maintained **exactly** by `SubmitToShard` / `CompleteShard`, which increment it on a +shard's `0→1` in-flight transition and decrement it on the `1→0` transition — so the divisor +always reflects the live set of concurrently-submitting shards, with no global in-flight counter +and no periodic reconciliation. Once more submitter threads than `NumShards` are active they +collide on the fixed shard set, so this counts occupied shards rather than distinct threads; the +shard count is sized so that stays a close proxy for concurrent submitter count. + +Because `perThread` saturates at `MaxPerThreadInFlight`, device-wide in-flight is bounded by +`NumShards × MaxPerThreadInFlight` = **4096** (on hosts with ≥ 16 logical processors) +independently of `T`. `T` therefore controls in-flight only in the `0 < T ≤ 4096` range; the +default `T = 4096` already sits at that ceiling. Raising the ceiling would mean growing +`SlotsPerShard` (and with it `MaxResults`), not raising `--device-throttle-limit`. + +`T` is a **coarse** bound, not an exact global cap. A shard admits against whatever divisor was +in effect at the time, and keeps that budget until it drains, so shards that filled while few +were occupied hold more than the final `T / active` share. Worst case is shards becoming +occupied one at a time with no completions in between: for `T = 120` across 32 shards the +aggregate settles near `Σ(⌊120/k⌋ + 1) ≈ 510`, roughly 4× the configured value. Exact +kernel-queue safety does not depend on this — it is enforced downstream by the native ring-full +retry (a submit that finds the ring full unwinds to `Pending` and retries after a completion). + +## Internal constants + +These are compile-time constants in `NativeStorageDevice.cs` (and `kMaxEvents` in +`file_linux.h`). They bound the formulas above. They are **not** knobs — each has a single +correct regime — but they define the tuning envelope. + +| Constant | Value | Role | +|---|---|---| +| `DefaultQueueDepth` | `4096` (1<<12) | default per-ring depth `D`. | +| `MaxQueueDepth` | `32768` (1<<15) | io_uring hard **ceiling** for `D`. | +| `DefaultThrottleLimit` | `4096` (1<<12) | default aggregate in-flight `T` for the Native device. | +| `kMaxEvents` (native) | `128` | **floor** for a libaio ring's native depth when the caller passes no explicit positive depth. | +| `LibaioReservationHeadroom` | `2` | over-provision multiplier on the libaio throttle-share depth. | +| `LibaioReservationFloor` | `128` (1<<7) | **floor** for the default libaio reservation depth. | +| `LibaioReservationCap` | `2048` (1<<11) | **cap** on a single libaio ring's reservation. | +| `DefaultAioMaxDevices` | `32` (1<<5) | default value of `AioMaxDevices` (the `--device-aio-max-devices` divisor). | +| `AioMaxDevices` | `32` | **process-wide** `public static int`. `fs.aio-max-nr` is machine-global, so "how many devices to fit within it" is a process policy, not per-device config — a static means devices created off the raw factory (cluster aux logs) honor the budget without plumbing. | +| `NumShards` | `Math.Min(2 × ProcessorCount, 32)` | per-submitter-thread shard count for in-flight de-contention. Two shards per core, capped at 32; the knee tracks peak concurrent submitters and is throttle-bounded past that. | +| `SlotsPerShard` | `256` | completion-slot free-list size per shard. | +| `MaxPerThreadInFlight` | `128` (`SlotsPerShard / 2`) | per-thread in-flight clamp; half of `SlotsPerShard` so a shard's free-list keeps 2× headroom and never empties under the throttle. | +| `ShardCounter` | `128 B` | cache-line-pair-padded per-shard in-flight counter struct (each shard's counter owns its own line, preventing false sharing). | +| `MaxResults` | `NumShards × 256` | size of the completion-context slot table (pure managed memory). | + +## Floor, cap, ceiling, headroom {#floor-cap-ceiling-headroom} + +The reservation math uses four distinct bounding concepts: + +* **Headroom** — a **multiplicative over-provision factor applied *above* a computed + need**. `LibaioReservationHeadroom = 2` sizes each libaio ring to *twice* its expected + steady-state in-flight (`share`). It is not a limit; it is slack so a transient burst of + reads does not momentarily fill the ring (which costs a ~2% ring-full IOPS dip). + +* **Floor** — a **lower bound**: the value is not sized *below* it by the share math. + `LibaioReservationFloor = 128` (and the native `kMaxEvents = 128`) keeps a ring able to hold + a minimum useful burst even when the throttle-share math computes something tiny (e.g. a high + ring count dividing a modest throttle), which would otherwise produce rings shallow enough to + stall constantly. A ceiling still overrides it: the per-device budget loop runs last and + halves below the floor when the reservation does not fit (e.g. 32 rings on a stock 65536 + budget resolve to `64`), because exceeding the budget fails device creation outright while a + shallow ring only costs throughput. + +* **Cap** — an **upper bound that is a self-imposed *policy* choice**. `LibaioReservationCap = + 2048` says a *single* libaio ring never reserves the full deep queue (`4096`) from the + global budget, because no single ring needs that much in-flight and the remainder stays + available to other coexisting devices. Exceeding it breaks nothing physically. + +* **Ceiling** — an **upper bound imposed by a *hard external constraint*** (kernel limit, + hardware, or a shared global budget), not a policy preference. Violating it is a hard + failure. Three appear in the math: + * `ceilingDepth` = the resolved `--device-queue-depth` — never reserve more than the ring is + actually sized for. + * `MaxQueueDepth = 32768` — io_uring's hard kernel maximum entries per ring. + * `perDeviceBudget = fs.aio-max-nr / AioMaxDevices` — the machine-global libaio budget + divided by the device target; exceeding it makes `io_setup` fail with `EAGAIN`. + +`ResolveLibaioReservationDepth` applies them in order: compute `share`, multiply by +**headroom**, raise to the **floor**, lower to the policy **cap**, lower to the queue-depth +**ceiling**, then lower again to the hard global-budget **ceiling**. + +## Tuning recipes + +Start from the defaults — on x64 Linux they reach peak out of the box: the smart io-context +default sizes rings to the hardware, and the Native throttle default of 4096 keeps NVMe +queues full. Only reach for the knobs below for a specific reason. + +* **io_uring, high connection count.** Set `--device-io-contexts` ≥ your peak concurrent + connections (e.g. `96` or `128`). The default caps at 64 rings; more connections than that + want more rings to stay 1:1 and contention-free. +* **libaio.** Leave `--device-io-contexts` at the default (ring-count-neutral). On a host left + at the stock `fs.aio-max-nr` of 65536 the per-device budget ceiling caps each device at 2048 + events, halving the default 4096 throttle — worth ~9% on a disk-serving workload, and silent + (no error). Size the budget so `fs.aio-max-nr / --device-aio-max-devices ≥` your throttle + (`sysctl -w fs.aio-max-nr=1048576`, persisted under `/etc/sysctl.d/` so it survives reboot). + If you run **many** Native devices in one process (cluster with many shards/AOF/checkpoint + logs) and hit `io_setup` `EAGAIN`, either raise `fs.aio-max-nr` or raise + `--device-aio-max-devices` so each device reserves a smaller slice of the budget + (`perDeviceBudget = fs.aio-max-nr / device-aio-max-devices`). The reservation guard logs an + actionable warning when `N × D` exceeds `fs.aio-max-nr`, and when a device's own reservation + cannot be brought within its per-device share (one event per ring is the floor). +* **Memory-constrained host.** Lower `--device-queue-depth` (e.g. `1024`) to cut io_uring ring + memory ~4× (uring ring memory ≈ `N × D × ~100 B`), and/or lower `--device-throttle-limit` to + cut pinned read-buffer memory (`T ×` read size). Both reduce a ceiling, not steady-state work. +* **Latency-sensitive over throughput.** Fewer, shallower rings and a lower throttle reduce + queueing depth at the cost of peak IOPS. + +## Diagnostics + +* Watch `iostat -x 1` on the tiered mount: at the serving peak the device queue (`aqu-sz`) + should be deep and `%util` ~100%. A shallow queue with idle device indicates the throttle or + ring depth is too low (or, for io_uring, too few rings serializing submitters). +* A startup `TsavoriteException` / `DllNotFoundException` naming `fs.aio-max-nr`, + `io_uring_disabled`, seccomp, or an old kernel is the native init surfacing an actionable + cause; a missing `liburing.so.2` is the most common load failure. + +## Related + +* [Configuration](configuration.md) — how all Garnet settings are parsed and applied. +* [Storage layer (Tsavorite)](tsavorite/intro.md) — the hybrid-log allocator that drives the device. diff --git a/website/docs/dev/tsavorite/buffer-pool.md b/website/docs/dev/tsavorite/buffer-pool.md index c9f504f6cf8..c4a35a0428c 100644 --- a/website/docs/dev/tsavorite/buffer-pool.md +++ b/website/docs/dev/tsavorite/buffer-pool.md @@ -113,51 +113,44 @@ The **depot** is the third tier — a shared fallback owned by the pool. Decodin (unlike shards/buckets, which are per-thread). * **per-class** — it is logically one depot *per size-class*. The backing array is laid out as `depot[cls * DepotStripes + stripe]`, so buffers of different sizes never mix. -* **lock-striped (8 stripes)** — within each class the depot is split into `DepotStripes = 8` independent - sub-stacks. A thread picks its stripe by `ThreadId & 7`. This is classic **lock striping** (8 locks instead of - 1), so several threads can push/pop concurrently without colliding. +* **lock-striped** — within each class the depot is split into `DepotStripes` independent sub-stacks. A thread + picks its stripe by `ThreadId & (DepotStripes - 1)`. This is classic **lock striping** (`DepotStripes` locks + instead of 1), so several threads can push/pop concurrently without colliding. The count is + `ConcurrencySharding.DepotStripeCount` — `2 × ProcessorCount` rounded up to a power of two, floored at 8 and + capped at 64 — so the number of locks scales with the number of threads that can contend for them. * **locked** — each stripe (`DepotStripe`) is a plain `Stack` guarded by a `lock` (Monitor). - A lock is acceptable here because the depot is the **cold path** — only reached when a thread's own local *and* - cross-thread lists are both empty (on `Get`), or when a thread's local cache is over its cap (on `Return`). A - lock is simpler than lock-free and avoids ABA entirely. - - A `ConcurrentStack` would not work here, for three reasons: + The depot is the **cold path** — reached only when a thread's own local *and* cross-thread lists are both empty + (on `Get`), or when a thread is at its local byte ceiling (on `Return`). Holding the lock across each stripe + operation buys three properties the depot depends on: 1. **Atomic close.** `Close()` sets the `closed` flag *and* drains the stripe under the one lock, so a push can - never land in a stripe that has already been drained. Lock-free, such a push would strand that buffer's byte - permit for the life of the pool — nothing ever revisits a closed stripe (§10). - 2. **Bounded capacity.** `ConcurrentStack` has no bounded form, so `DepotStripeCap` would need a separate - interlocked counter that can drift from the actual contents. That cap is what bounds how much memory threads - may park cross-thread, so it must be enforced atomically with the push. - 3. **No per-push allocation.** `ConcurrentStack.Push` allocates a link node per item, which is exactly what this - pool exists to avoid; `Stack` pushes into a pre-grown array. - - Contention is a non-issue: the depot is already striped 8 ways, each critical section is O(1), and a lock-free - CAS on a single shared head would reintroduce the cross-core cache-line ping-pong this design removes. + never land in a stripe that has already been drained and strand that buffer's byte permit — nothing ever + revisits a closed stripe (§10). + 2. **Bounded capacity.** `DepotStripeCap` is tested and the push applied under the same lock, so the bound is + exact rather than an approximation that can drift from the stripe's actual contents. + 3. **No per-push allocation.** `Stack` pushes into a pre-grown array, so parking a buffer allocates nothing. + + Contention is bounded by striping: the depot is spread `DepotStripes` ways, sized from the machine's processor + count, and each critical section is O(1). * **overflow pool** — it catches buffers that cannot stay in per-thread caches, and redistributes them: - * On **Return**, if the origin thread's local stack is full (over `LocalCap`/`LocalByteCap`) or the origin - shard has retired (its thread died), the buffer overflows into the depot (`DepotPush`) instead of being - dropped. + * On **Return**, if the origin thread is at its local byte ceiling (§9) or the origin shard has retired (its + thread died), the buffer overflows into the depot (`DepotPush`) instead of being dropped. * On **Get**, after checking its own local and cross-thread lists, a thread pulls from the depot (`DepotPop`) - before allocating fresh. `DepotPop` scans all 8 stripes starting at the thread's own — a cheap form of - **work-stealing** so buffers parked by a now-idle thread get reused by an active one. + before allocating fresh. `DepotPop` scans every stripe of the class, starting at the thread's own — a cheap + form of **work-stealing** so buffers parked by a now-idle thread get reused by an active one. ### Large classes are depot-only -The per-thread local/cross-thread tiers are ideal for the **small, hot** record-sized buffers a thread reuses -back-to-back. They are the *wrong* place for **large** buffers (record/flush reads above `LargeTierMinBytes`, -256 KB): under a wide value-size mix a thread rarely re-requests the same large class twice in a row, so a large -buffer parked on its origin thread mostly sits idle — and with many threads × many large classes this **strands** -big buffers and multiplies the working set, inflating peak RSS and Gen2 GC even though the byte budget is honored. +The per-thread tiers exploit a thread reusing the same size back-to-back, which is the access pattern of the +**small, hot** record-sized buffers. **Large** buffers (record/flush reads above `LargeTierMinBytes`, 256 KB) do +not have it: under a wide value-size mix a thread rarely re-requests the same large class twice in a row, so a +large buffer parked on its origin thread sits idle while the working set grows as (threads × large classes). So on `Return`, **large-class buffers skip the per-thread tiers entirely and go straight to the shared depot**, on -*both* the owner-return and foreign-return paths (`page.Level >= firstLargeClass` in `ReturnOriginReturn`). Any -thread can then reuse them from the depot under the same `largeBudget`, which recovers legacy-like sharing for big -buffers while small classes keep the atomic-free per-thread fast path. Because large-buffer operations are -low-rate and the disk workload is bandwidth-bound, the striped-depot lock handoff costs effectively nothing. -Routing large owner-returns through the pool-owned depot also sidesteps the origin-shard finalize race by -construction. Empirically, at the default 1 GiB budget under a 100 B–10 MB disk read mix at 64 threads, this -raised large-class reuse from ~12–50 % to ~54–96 %, cut new allocations and Gen2 collections by roughly half, and -lowered peak RSS — with no throughput change. +*both* the owner-return and foreign-return paths (`page.Level >= firstLargeClass` in `ReturnOriginReturn`). One +global working set of large buffers then serves every thread under the same `largeBudget`, while small classes +keep the atomic-free per-thread fast path. Large-buffer operations are low-rate and the disk workload is +bandwidth-bound, so the striped-depot lock handoff costs effectively nothing. Routing large owner-returns through +the pool-owned depot also keeps them clear of the origin-shard finalize race by construction. ## 7. The size-class ladder @@ -175,8 +168,8 @@ exact-fit buffer that is never pooled and goes straight back to the GC on `Retur inverse (class → capacity). Both functions are pure O(1) integer math with no table scans. The geometric region emits **two** classes per doubling — a midpoint at `1.5 × 2^octave` and a top at -`2^(octave+1)` — which is why `NumClasses` carries a `2 * GeometricDoublings` term. The second class per doubling -is what bounds worst-case over-allocation at 1.5× rather than the 2× a single class per doubling would give. +`2^(octave+1)` — which is why `NumClasses` carries a `2 * GeometricDoublings` term, and which bounds worst-case +over-allocation at 1.5×. The 16 MB ceiling is chosen so that a record built on a multi-MB inline value up to nearly Garnet's ~16 MB maximum is pooled rather than bypassed, and so the 4 MB object-log flush buffer lands exactly on a class with @@ -193,8 +186,8 @@ fastest first: own buffers. 2. **Cross-thread stack** — if local is empty, bulk-claim `crossThreadHead` with one CAS, keep one buffer, splice the rest into local. This is where buffers that other threads returned to me come home. -3. **Depot** — if both are empty, `DepotPop(cls)` across the 8 stripes. For **large** classes the first two tiers - are always empty (large buffers are returned straight to the depot — see §6), so this is their normal source. +3. **Depot** — if both are empty, `DepotPop(cls)` across the class's stripes. For **large** classes the first two + tiers are always empty (large buffers are returned straight to the depot — see §6), so this is their normal source. 4. **Allocate** — `AllocateForBucket`: allocate a new `byte[]`, compute alignment, and try to reserve a budget **permit** (see §9). If the reservation succeeds, the buffer is marked `cacheable`; otherwise it is still served to the caller but marked non-cacheable and dropped on `Return`. @@ -229,13 +222,32 @@ is partitioned into **two independent `BudgetState` instances**: capacity exceeds it draw from `largeBudget`; everything else draws from `smallBudget`. A flood of large record or flush buffers can, at worst, exhaust the large slice; the small slice is reserved for the hot path. -### Local retention caps +### Per-thread local retention + +Local retention is bounded in **bytes per thread**, not in buffers per class: one class's buffer is up to 512× +another's, and the buffer count a thread needs is its in-flight I/O pipeline depth — a property of the caller +rather than of the pool. Each thread's ceiling (`ThreadShard.localByteCap`) is an equal slice of the sub-budget +that local caching draws from: + +``` +threadLocalByteCap = max(smallBudget / ExpectedConcurrentThreads, MinThreadLocalBytes) +``` + +`ExpectedConcurrentThreads` is `2 × ProcessorCount` capped at 64, and `MinThreadLocalBytes` (1 MB) is a floor so +a small configured budget still caches. At the 1 GiB default on a 32-core or larger box that is +256 MB / 64 = **4 MB** per thread. The slices of that many threads sum to the sub-budget, so no thread can retain +enough to starve the others. Only small classes reach this path; large classes go straight to the depot (§6). + +The ceiling is shared across all of a thread's size classes and is enforced only once the thread reaches it, so a +thread whose traffic is a single class keeps the whole slice. At the ceiling, admission is **max-min fair** across +the classes that thread actually uses (`TryMakeRoom`): a class holding less than an equal share +(`localByteCap / activeClasses`) is admitted, and room is made by spilling the class furthest *above* its share +(`WorstOverShare` → `SpillOneLocal`). A class whose chain is empty still counts itself active, so a starved class +is not shut out by the classes crowding it. -`LocalCap` (128 buffers) and `LocalByteCap` (32 MB) bound a single `(thread, size-class)` **local stack** only — -they govern placement/locality, not the global budget. When a local stack exceeds either cap on `Return`, the -buffer spills to the shared depot (relocated, not dropped — it still counts against the byte budget). These caps -stop one thread from hoarding an unbounded number, or an unbounded byte-size, of buffers for a single class while -other threads starve. +A spill is a **relocation, not an eviction**: the buffer moves to the shared depot, where any thread can still +reuse it, and its permit travels unchanged. The cap therefore selects *where* a buffer is cached, never *whether* — +only budget exhaustion makes a buffer uncacheable. ## 10. Lifecycle and correctness @@ -278,7 +290,8 @@ freeing thread is not the allocating thread, each **(pool, thread)** gets a priv **buckets**; each bucket has an atomic-free **local stack** (owner reuse) and a lock-free MPSC **cross-thread stack** (buffers routed home to their *origin* thread). **Small** classes use those per-thread tiers as the hot path; **large** classes (> 256 KB) bypass them and share globally through the depot. Overflow and cross-thread -redistribution go through a shared, per-class, 8-way lock-striped **depot**. A per-pool **byte budget**, split -into isolated small and large slices, caps total retained bytes via one permit per buffer taken at birth and -released at death. Seal sentinels, finalizers, and a closing state machine make thread death and pool teardown -race-safe. +redistribution go through a shared, per-class, lock-striped **depot** (8–64 stripes, sized from the processor +count). A per-pool **byte budget**, split into isolated small and large slices, caps total retained bytes via one +permit per buffer taken at birth and released at death; within the small slice each thread retains up to an equal +per-thread byte share, spilling the surplus to the depot rather than dropping it. Seal sentinels, finalizers, and +a closing state machine make thread death and pool teardown race-safe. diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index a1bb8ae50b7..8fa69fd74bd 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -202,7 +202,12 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **DeviceType** | ```--device-type``` | ```DeviceType``` | Default, Native, RandomAccess, FileStream, AzureStorage, LocalMemory, Null | Device type (Default, Native, RandomAccess, FileStream, AzureStorage, LocalMemory, Null) | | **DeviceIoBackend** | ```--device-io-backend``` | ```IoBackend``` | Default, Libaio, Uring | Linux-only IO backend for DeviceType=Native: Default (=libaio), Libaio, or Uring (io_uring). The shipped native library is built with -DUSE_URING=ON and requires liburing.so.2 at load time for all backends; a -DUSE_URING=OFF rebuild only needs libaio. | | **DeviceCompletionThreads** | ```--device-completion-threads``` | ```int``` | Integer in range:
[1, 64] | Linux-only: Number of IO completion drain threads for DeviceType=Native (default 4, max 64). Under high concurrent pending-read load a single drainer convoys on the completion-signal path and collapses throughput; 4 removes that on both backends. On io_uring this scales further/more CPU-efficiently; libaio benefits less beyond a few. | -| **DeviceThrottleLimit** | ```--device-throttle-limit``` | ```int``` | Integer in range:
[0, 65536] | Per-device max number of in-flight IOs (IDevice.ThrottleLimit). 0 = use the device's built-in default (120 for the in-box Tsavorite devices). Raising this lets disk-bound workloads keep the queue depth high enough to saturate fast NVMe / io_uring backends. For DeviceType=LocalMemory (which has no device-wide throttle) this instead sets the per-ring in-flight capacity, rounded up to a power of two. | +| **DeviceThrottleLimit** | ```--device-throttle-limit``` | ```int``` | Integer in range:
[0, 65536] | Per-device max number of in-flight IOs (IDevice.ThrottleLimit). 0 = use the device's built-in default (4096 for DeviceType=Native; 120 for the other in-box Tsavorite devices such as RandomAccess/FileStream). Raising this lets disk-bound workloads keep the queue depth high enough to saturate fast NVMe / io_uring backends. For DeviceType=LocalMemory (which has no device-wide throttle) this instead sets the per-ring in-flight capacity, rounded up to a power of two. | +| **DeviceIoContexts** | ```--device-io-contexts``` | ```int``` | Integer in range:
[0, 4096] | Linux-only, DeviceType=Native: number of independent kernel io_contexts / io_uring rings (ring COUNT), decoupled from --device-completion-threads. Critical for io_uring: set at or above submitter concurrency (roughly your connection count) so each submitter owns a ring and io_submit is contention-free; too few rings serialize submitters on a per-ring lock and cost up to ~3x. libaio is largely indifferent. 0 = device default. | +| **DeviceQueueDepth** | ```--device-queue-depth``` | ```int``` | Integer in range:
[0, 32768] | Linux-only, DeviceType=Native: per-ring kernel submission depth (maxEvents for io_uring_queue_init / libaio io_setup). Orthogonal to --device-io-contexts (ring count) and --device-throttle-limit (aggregate in-flight). 0 = device default. Note: for libaio, io-contexts x queue-depth is drawn from the global fs.aio-max-nr budget. | +| **DeviceUringSqPoll** | ```--device-uring-sqpoll``` | ```bool``` | | Linux-only, DeviceType=Native + --device-io-backend=Uring: enable io_uring SQPOLL (IORING_SETUP_SQPOLL) so a kernel thread polls the submission queue and submissions are syscall-free. Each ring gets its own poll thread, so submission stays parallel across rings. Off by default (opt-in); ignored for libaio. Busy-polling kernel threads consume CPU, so benchmark it against the default per-submit path. | +| **DeviceUringSqPollIdleMs** | ```--device-uring-sqpoll-idle-ms``` | ```int``` | Integer in range:
[0, 600000] | io_uring SQPOLL poll-thread idle window in milliseconds (sq_thread_idle): how long the kernel poll thread spins after the last submit before parking. 0 = native default (10s). Only meaningful with --device-uring-sqpoll. | +| **DeviceAioMaxDevices** | ```--device-aio-max-devices``` | ```int``` | Integer in range:
[1, 4096] | Linux-only, DeviceType=Native (libaio): target number of Native devices to fit within the machine-global fs.aio-max-nr libaio budget (default 32). libaio io_setup permanently reserves io-contexts x queue-depth events from that global budget per device, so the default per-device reservation is capped at fs.aio-max-nr / this. To recover from io_setup EAGAIN, raise fs.aio-max-nr or raise this (a larger divisor shrinks each device's reservation); lower this to give each device a deeper reservation. Ignored for io_uring (no global budget) and non-Linux. | | **RevivBinRecordSizes** | ```--reviv-bin-record-sizes``` | ```IEnumerable``` | | #,#,...,#: The sizes of records in each revivification bin, in order of increasing size. Supersedes the default --reviv; cannot be used with --reviv-in-chain-only | | **RevivBinRecordCounts** | ```--reviv-bin-record-counts``` | ```IEnumerable``` | | #,#,...,#: The number of records in each bin: Default (not specified): If reviv-bin-record-sizes is specified, each bin is 256 records # (one value): If reviv-bin-record-sizes is specified, then all bins have this number of records, else error #,#,...,# (multiple values): If reviv-bin-record-sizes is specified, then it must be the same size as that array, else error Supersedes the default --reviv; cannot be used with --reviv-in-chain-only | | **RevivifiableFraction** | ```--reviv-fraction``` | ```double``` | Double in range:
[0, 1] | #: Fraction of mutable in-memory log space, from the highest log address down to the read-only region, that is eligible for revivification. | @@ -239,3 +244,23 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **VectorSetQuantizationTaskCount** | ```--vector-set-quantization-task-count``` | ```int``` | Integer in range:
[0, MaxValue] | Configure how many quantization tasks are used to optimize Vector Set operations (default: 0 uses the machine CPU count; maximum: the machine CPU count) | [^1]: A string representing a memory size. Can either be a number of bytes, or follow this pattern: 1k, 1kb, 5M, 5Mb, 10g, 10GB etc. + +--- + +## Native device IO tuning (Linux) + +When `--device-type Native` is used on Linux (the default on x64 Linux), four orthogonal knobs +size the kernel IO path. Each controls exactly one dimension: + +| Knob | Controls | Guidance | +| --- | --- | --- | +| `--device-io-backend` | `Libaio` (default) or `Uring` (io_uring) | Both reach the same peak on fast NVMe when sized correctly. | +| `--device-io-contexts` | Ring **count** (independent io_uring rings / libaio io_contexts) | **Critical for io_uring:** set at or above submitter concurrency (~your connection count) so each submitter owns a ring and submission is lock-free; too few rings serialize submitters on a per-ring lock and can cost ~3x. **libaio is largely indifferent** (its kernel io_context mutex is cheap), so it needs no tuning here. | +| `--device-queue-depth` | Per-ring **depth** (kernel submission queue size) | Independent of ring count. For **libaio**, `io-contexts × queue-depth` is drawn from the global `fs.aio-max-nr` budget, so keep the product within that limit (raise `fs.aio-max-nr`, or lower one factor, if device creation fails). io_uring depth is per-ring memory only. | +| `--device-completion-threads` | Number of background completion (drain) threads | Default 4 is sufficient on both backends; a single drainer convoys under high pending-read load. | +| `--device-throttle-limit` | Aggregate **in-flight** reads before backpressure | Software backpressure only — it bounds pinned read-buffer memory and must stay `≤ io-contexts × queue-depth`. Raise it for very high connection counts; the default saturates fast NVMe at typical thread counts. | + +Mental model: `io-contexts` = number of submission queues, `queue-depth` = depth of each queue, so +`io-contexts × queue-depth` is the kernel capacity; `throttle-limit` is how much of that capacity the +allocator keeps in flight at once. The most common tuning mistake is running the **io_uring** backend +with too few rings — always set `--device-io-contexts` to cover your submitter concurrency for io_uring. diff --git a/website/sidebars.js b/website/sidebars.js index c8f0c1f9df5..5150c985b6e 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -25,6 +25,7 @@ const sidebars = { {type: 'category', label: 'Cluster Mode', items: ["cluster/overview", "cluster/replication", "cluster/key-migration"]}, {type: 'category', label: 'Developer Guide', items: ["dev/onboarding", "dev/code-structure", "dev/configuration", "dev/network", "dev/processing", "dev/garnet-api", {type: 'category', label: 'Tsavorite - Storage Layer', collapsed: true, items: ["dev/tsavorite/intro", "dev/tsavorite/reviv", "dev/tsavorite/locking", "dev/tsavorite/readcache", "dev/tsavorite/storefunctions", "dev/tsavorite/epochprotection", "dev/tsavorite/logrecord", "dev/tsavorite/object-allocator", "dev/tsavorite/buffer-pool"]}, + "dev/device-tuning", "dev/transactions", "dev/custom-commands", "dev/multi-db",