[Storage] Optimize IOPS for RAID-0 NVMe disks

.github/workflows/native-build.yml

@@ -91,6 +91,23 @@ jobs:
           ls -la "staging/${{ matrix.rid }}/native"
           file "staging/${{ matrix.rid }}/native"/*.so
 
+      - name: Verify required exports
+        run: |
+          set -euo pipefail
+          so="staging/${{ matrix.rid }}/native/libnative_device.so"
+          # The C# NativeStorageDevice loader hard-probes these at device creation: ImportResolver
+          # binds the P/Invokes and the startup ABI probe calls NumIoContexts / QueueRunFor /
+          # TryCompleteMine (the affine inline-drain path is on by default). CreateWithBackend is the
+          # create export the managed wrapper binds; a binary missing any of these throws at server
+          # startup on this RID, so fail the build here instead of shipping it.
+          # Host nm reads the ELF dynamic symbol table for every RID (x64/arm64, glibc/musl) alike.
+          command -v nm >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq binutils; }
+          rc=0
+          for sym in NativeDevice_CreateWithBackend NativeDevice_NumIoContexts NativeDevice_QueueRunFor NativeDevice_TryCompleteMine; do
+            if nm -D --defined-only "$so" | grep -qw "$sym"; then echo "  OK      $sym"; else echo "  MISSING $sym"; rc=1; fi
+          done
+          [ $rc -eq 0 ] || { echo "::error::$so is missing a required NativeDevice_* export the C# loader probes at startup"; exit 1; }
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: native-${{ matrix.rid }}
@@ -179,10 +196,18 @@ jobs:
           $dll = "staging/${{ matrix.rid }}/native/native_device.dll"
 
           $exports = & $dumpbin /exports $dll
-          if (-not ($exports | Select-String 'NativeDevice_CreateWithBackend')) {
-            throw "native_device.dll is missing NativeDevice_CreateWithBackend export"
+          # The C# NativeStorageDevice loader hard-probes these at device creation: ImportResolver
+          # binds the P/Invokes and the startup ABI probe calls NumIoContexts / QueueRunFor /
+          # TryCompleteMine (the affine inline-drain path is on by default). CreateWithBackend is the
+          # create export the managed wrapper binds; a binary missing any of these throws at server
+          # startup on this RID, so fail the build here instead of shipping it.
+          $required = @('NativeDevice_CreateWithBackend','NativeDevice_NumIoContexts','NativeDevice_QueueRunFor','NativeDevice_TryCompleteMine')
+          foreach ($sym in $required) {
+            if (-not ($exports | Select-String -SimpleMatch $sym)) {
+              throw "native_device.dll is missing required export $sym"
+            }
           }
-          Write-Host "Exports verified."
+          Write-Host "Exports verified: $($required -join ', ')."
 
           # Confirm the security flags in CMakeLists (/guard:cf and /GS /sdl, set alongside /Qspectre)
           # actually took effect in the produced binary: Control Flow Guard and the stack Security

benchmark/Resp.benchmark/OfflineBench/ReqGen.cs

@@ -15,7 +15,7 @@ public unsafe partial class ReqGen
         static int bitfieldOpCount = 3;
 
         readonly byte[][] buffers;
-        readonly List<List<string>> flatRequestBuffer;
+        readonly List<string[]> flatRequestBuffer;
         readonly int[] lens;
         readonly OpType opType;
         readonly bool randomGen, randomServe;
@@ -67,7 +67,7 @@ public ReqGen(
             }
             buffers = new byte[NumBuffs][];
 
-            flatRequestBuffer = flatBufferClient ? new List<List<string>>() : null;
+            flatRequestBuffer = flatBufferClient ? new List<string[]>() : null;
             lens = new int[NumBuffs];
             BatchCount = BatchSize;
             this.opType = opType;
@@ -114,19 +114,19 @@ public byte[] GetRequest(out int len)
             int offset;
 
             if (randomServe)
-                offset = r.Next(NumBuffs);
+                offset = Random.Shared.Next(NumBuffs);
             else
                 offset = (Interlocked.Increment(ref seqNo) - 1) % NumBuffs;
 
             len = lens[offset];
             return buffers[offset];
         }
 
-        public List<string> GetRequestArgs()
+        public string[] GetRequestArgs()
         {
             int offset;
             if (randomServe)
-                offset = r.Next(flatRequestBuffer.Count);
+                offset = Random.Shared.Next(flatRequestBuffer.Count);
             else
                 offset = (Interlocked.Increment(ref seqNo) - 1) % flatRequestBuffer.Count;
 
@@ -150,9 +150,7 @@ private void ConvertToSERedisInput(OpType opType)
                     case OpType.GET:
                     case OpType.SET:
                     case OpType.MSET:
-                        var buffer = buffers[i];
-                        flatRequestBuffer.Add(new List<string>());
-                        ProcessArgs(i, buffer);
+                        ProcessArgs(buffers[i]);
                         break;
                     default:
                         Console.WriteLine($"op {opType} not supported with SERedis! Skipping conversion to SERedis input!");
@@ -161,19 +159,21 @@ private void ConvertToSERedisInput(OpType opType)
             }
         }
 
-        private void ProcessArgs(int i, byte[] buffer)
+        private void ProcessArgs(byte[] buffer)
         {
             fixed (byte* buf = buffer)
             {
                 byte* ptr = buf;
                 RespReadUtils.TryReadUnsignedArrayLength(out int count, ref ptr, buf + buffer.Length);
-                RespReadUtils.TryReadStringWithLengthHeader(out var cmd, ref ptr, buf + buffer.Length);
 
-                for (int j = 0; j < count - 1; j++)
-                {
-                    RespReadUtils.TryReadStringWithLengthHeader(out var arg, ref ptr, buf + buffer.Length);
-                    flatRequestBuffer[i].Add(arg);
-                }
+                // Keep the command token (e.g. "MSET") as element 0 so the cached array is a complete,
+                // ready-to-send argument list. Serve-side clients pass it straight to Execute with no
+                // per-request allocation, copy, or command prepend; consumers that only want the payload
+                // (e.g. SERedis key/value pairs) start reading at index 1.
+                var args = new string[count];
+                for (int j = 0; j < count; j++)
+                    RespReadUtils.TryReadStringWithLengthHeader(out args[j], ref ptr, buf + buffer.Length);
+                flatRequestBuffer.Add(args);
             }
         }
 

benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs

@@ -295,7 +295,7 @@ public void Run(
                 rg = run_rg;
             else
             {
-                rg = new ReqGen(Start, opts.DbSize, TotalOps, BatchSize, opType, randomGen, randomServe, keyLen, valueLen, ttl: ttl);
+                rg = new ReqGen(Start, opts.DbSize, TotalOps, BatchSize, opType, randomGen, randomServe, keyLen, valueLen, flatBufferClient: (opts.Client == ClientType.SERedis || opts.Client == ClientType.GarnetClientSession), ttl: ttl);
                 rg.Generate();
             }
 
@@ -476,11 +476,11 @@ private void GarnetClientSessionOperateThreadRunner(int NumOps, OpType opType, R
 
             Stopwatch sw = new();
             sw.Start();
+            // GetRequestArgs returns a shared, cached argument array that already includes the "MSET" command
+            // token at index 0, so it is sent directly with no per-request allocation, copy, or mutation.
             while (!done)
             {
-                var reqArgs = rg.GetRequestArgs();
-                reqArgs.Insert(0, "MSET");
-                c.Execute([.. reqArgs]);
+                c.Execute(rg.GetRequestArgs());
                 c.CompletePending(true);
                 numReqs++;
                 if (numReqs == maxReqs) break;
@@ -511,7 +511,8 @@ private void SERedisOperateThreadRunner(int NumOps, OpType opType, ReqGen rg)
             while (!done)
             {
                 var reqArgs = rg.GetRequestArgs();
-                for (var i = 0; i < reqArgs.Count; i += 2)
+                // Index 0 is the command token ("MSET"); key/value pairs start at index 1.
+                for (var i = 1; i < reqArgs.Length; i += 2)
                     db.StringSet(reqArgs[i], reqArgs[i + 1]);
                 numReqs++;
                 if (numReqs == maxReqs) break;

benchmark/Resp.benchmark/Options.cs

@@ -53,6 +53,9 @@ public partial class Options
         [Option('t', "threads", Separator = ',', Default = new[] { 1, 2, 4, 8, 16, 32 }, HelpText = "Number of threads (comma separated)")]
         public IEnumerable<int> NumThreads { get; set; }
 
+        [Option("load-threads", Required = false, Default = 8, HelpText = "Number of threads used for the initial data load phase")]
+        public int LoadThreads { get; set; }
+
         [Option('a', "auth", Required = false, Default = null, HelpText = "Authentication password")]
         public string Auth { get; set; }
 

benchmark/Resp.benchmark/Program.cs

@@ -333,7 +333,7 @@ static void RunBasicCommandsBenchmark(Options opts)
                 var bench = new RespPerfBench(opts, 0, redis);
 
                 if (!opts.SkipLoad)
-                    bench.LoadData(keyLen: keyLen, valueLen: valueLen, numericValue: opts.Op == OpType.INCR);
+                    bench.LoadData(loadDbThreads: opts.LoadThreads, keyLen: keyLen, valueLen: valueLen, numericValue: opts.Op == OpType.INCR);
 
                 // --runtime 0 seeds the keyspace only; skip the run phase.
                 if (opts.RunTime != 0)

benchmark/Resp.benchmark/README.md

@@ -36,7 +36,7 @@ Always measure on a **Release** build. `dotnet $RB --help` lists all flags.
 |---|---|---|
 | `--op` | `GET` | Op to benchmark (offline): GET, MGET, INCR, SET, ZADD, ... |
 | `--dbsize` | `1024` | Distinct keys (pre-loaded unless `-s`). |
-| `--valuelength` | `8` | Value bytes (use `100` for KV.benchmark parity). |
+| `--valuelength` | `8` | Value bytes (use `--keylength 16 --valuelength 96` = 128 B record for KV/Device parity). |
 | `-t` | `1,2,4,8,16,32` | Thread-count sweep (offline). |
 | `-b` | `4096` | Requests per pipeline (offline; dominant throughput knob, `1024` is a good default). Online forces `1`. |
 | `--runtime` | `15` | Seconds per cell. `0` = load only (no run). |
@@ -63,22 +63,54 @@ dotnet $RB -s --op GET --dbsize 16777216 --valuelength 100 -t 1,2,4,8,16,32 -b 1
 ### 2. NVMe storage-bound — reads hit real disk
 
 Tier the store with a tiny memory log so ~99.9% of a 100 M dataset is on NVMe and
-every GET is a 4 KB random fetch. Use **100 M × 100 B** (smaller datasets touch few
-NAND dies and understate device IOPS).
+every GET is a random device fetch. Use **100 M × 128 B** records (`--keylength 16
+--valuelength 96`, matching the KV/Device benchmarks — 128 B records read over the
+array's 512 B sectors). Reference host: **8×NVMe RAID-0** (`/raid`, `fio` random-read
+ceiling ≈ **8.24 M IOPS at 4 K** / **8.20 M at 512 B** — the array is IOPS-bound, so
+block size barely moves it); Garnet sustains **~7.3 M** end-to-end (≈ 89% of `fio`).
 
 ```bash
-DATA=/mnt/nvme/garnet; mkdir -p $DATA
-numactl --cpunodebind=0 --membind=0 dotnet $GS --port 6379 \
-  --memory 16m --page 4m --segment 1g --index 4g --storage-tier --logdir $DATA \
-  --device-type Native --device-io-backend libaio --device-throttle-limit 512 \
-  --logger-level Warning &
-numactl --cpunodebind=0 --membind=0 dotnet $RB --op MSET --dbsize 100000000 --valuelength 100 -t 8 -b 1024 --runtime 0
-numactl --cpunodebind=0 --membind=0 dotnet $RB -s --op GET --dbsize 100000000 --valuelength 100 -t 1,2,4,8,16,32 -b 1024 --runtime 15
+DATA=/raid/garnet; mkdir -p $DATA
+# Server pinned to NUMA node 0, client driven from node 1:
+numactl --cpunodebind=0 --membind=0 dotnet $GS --port 6379 --bind 127.0.0.1 \
+  --memory 16m --page 4m --segment 1g --index 8g --storage-tier --logdir $DATA \
+  --device-type Native --device-io-backend Libaio --logger-level Warning &
+numactl --cpunodebind=1 --membind=1 dotnet $RB --op MSET --dbsize 100000000 \
+  --keylength 16 --valuelength 96 --client LightClient --load-threads 32 -b 4096 --runtime 0
+numactl --cpunodebind=1 --membind=1 dotnet $RB -s --op GET --dbsize 100000000 \
+  --keylength 16 --valuelength 96 --client LightClient -t 8,32,48,64 -b 1024 --runtime 12
 ```
 
-- `--index 4g` for 100 M keys (default 128 m → 3–4× slowdown from hash chains).
-- `libaio` is fastest on Linux (`uring` to compare; `Default` → RandomAccess, slower).
-  `--device-throttle-limit 512` is safe on fast NVMe; lower to 128 on SATA.
+**Check the load before trusting the GET numbers.** A key that was never written is
+answered from memory without touching the device, and a miss is ~30× cheaper than a
+disk read, so a partial load silently inflates the result — an unloaded store reports
+>150 M ops/sec on this host. Confirm the `MSET` step reports ~100 M ops in its
+`[Total time]` line; the [generator script](scripts/nvme-raid0-matrix.sh) enforces this.
+
+| backend | NUMA | t=8 | t=32 | t=48 | t=64 |
+|---|---|---|---|---|---|
+| Libaio | srv node-0 / cli node-1 | 1.82 M | 5.70 M | 6.97 M | **7.06 M** |
+| Libaio | no pin | 1.38 M | 5.09 M | 5.72 M | 5.57 M |
+
+> `Libaio` (the Linux default) is shown here for a quick look. For the full
+> backend × pin matrix — including `Uring` on out-of-box defaults, which reaches a
+> slightly higher peak — see [Sample results](#sample-results--8-nvme-ssd-raid-0) below.
+
+- `--index 8g` for 100 M keys (default 128 m → 3–4× slowdown from hash chains).
+- Peak is in the **t=48–64** band: the RESP server's pipelined client connections drive
+  in-flight depth through the server's own network + completion threads, so throughput
+  keeps climbing well past the raw device's t=32 peak before queueing costs take over.
+- **NUMA pinning matters most here** (stateful server): pinning the server to node 0
+  and the client to node 1 lifts t=48 from 5.72 → 6.97 M and t=64 from 5.57 → 7.06 M.
+- **`Libaio`** is the Linux default and needs no ring tuning. **`Uring`** now auto-sizes
+  its ring count to `min(2 × cores, 64)` — decoupled from `--device-completion-threads` —
+  so it is competitive with libaio out of the box; use **`--device-io-contexts N`** to set
+  the ring count explicitly (at or above your connection count) for very high concurrency
+  (see [Device Tuning](https://microsoft.github.io/garnet/docs/dev/device-tuning) and the
+  [Device README](../../libs/storage/Tsavorite/cs/benchmark/Device.benchmark/README.md#nvme-storage-bound)).
+- Capacity knobs are left at their defaults above; add `--device-completion-threads 8
+  --device-throttle-limit 4096` to hand-tune. `--device-throttle-limit 4096` suits this
+  array; lower to 512/128 on a single/SATA disk.
 
 ### 3. Memory-device-bound — reads hit the in-RAM device
 
@@ -94,6 +126,62 @@ LocalMemory runs). Replace the device flags in (2) with:
 
 Reference (10 M × 100 B, t=16): **~2.7 M ops/sec** at `-b 1024`, **~3.7 M** at `-b 256`.
 
+## Sample results — 8× NVMe SSD RAID-0
+
+Full **scenario 2** GET throughput matrix on **out-of-box device defaults** — only `--storage-tier`
+and `--device-io-backend` are set; `--device-completion-threads`, `--device-throttle-limit`, and
+`--device-io-contexts` are left at their server defaults, so this is what an operator gets with zero
+device tuning. Median of 3 passes per cell.
+
+**Host** — 2× Intel Xeon Platinum 8480CL (56 cores × 2 threads/socket, 224 logical CPUs, 2 NUMA nodes),
+~2 TB DDR5; **8× Kioxia KCM6DRUL3T84** 3.84 TB PCIe-Gen4 NVMe in Linux `md` RAID-0 (`/dev/md1`, 512 KB
+chunks, ext4, ≈28 TB); Ubuntu 24.04.4 LTS, kernel 6.8.0-136, .NET 10.0.302; `fs.aio-max-nr` = 4194304.
+`fio` random-read ceiling on this array: **8.24 M IOPS at 4 K** and **8.20 M IOPS at 512 B**
+(32 jobs × QD64, io_uring, `O_DIRECT`, 8 files) — the array is IOPS-bound at these sizes, so the
+ceiling is effectively block-size independent.
+
+**Workload** — 100 M × 128 B records (`--keylength 16 --valuelength 96`) tiered onto the array
+(`--memory 16m --page 4m --segment 1g --index 8g`); 100% random GET; client `-b 1024`; 12 s per cell.
+The generator verifies that the load actually wrote ~100 M keys before it measures, so every cell
+below is storage-bound rather than partly served as in-memory misses.
+
+| backend | NUMA | t=8 | t=32 | t=48 | t=64 |
+|---|---|---|---|---|---|
+| Libaio | srv node-0 / cli node-1 | 1.82 M | 5.70 M | 6.97 M | 7.06 M |
+| Libaio | no pin | 1.38 M | 5.09 M | 5.72 M | 5.57 M |
+| Uring | srv node-0 / cli node-1 | 1.82 M | 6.13 M | **7.32 M** | 6.89 M |
+| Uring | no pin | 1.48 M | 4.60 M | 5.55 M | 6.29 M |
+
+- **Peak ≈ 7.3 M ops/sec** (uring, pinned, t=48) — **~89% of the `fio` ceiling** for random reads
+  of the same shape (128 B records fetched over the array's 512 B sectors), driven end-to-end
+  through the RESP protocol and the Tsavorite pending-read path (not raw device IO). The remaining
+  gap to `fio` is the RESP + pending-read path: the raw device layer reaches ~8.4 M on this array.
+- **The two backends peak at different thread counts.** Uring peaks at t=48 and eases off at t=64,
+  where added client concurrency costs more in queueing than it recovers in in-flight depth; libaio
+  is still climbing at t=64. Sweep the t=48–64 band rather than assuming a single best value.
+- **Defaults reach the tuned peak.** Uring's smart ring-count default (`min(2 × cores, 64)` rings,
+  decoupled from the 4 completion threads) sizes rings to the hardware with no flags; libaio
+  needs no ring tuning. Explicit tuning (`--device-completion-threads 8 --device-throttle-limit 4096`,
+  uring `--device-io-contexts 96`) moves each cell by only a few percent on this host.
+- **NUMA pinning is the largest single factor** on this dual-socket box (stateful server): e.g. uring
+  t=48 rises 5.55 → 7.32 M when the server is pinned to node 0 and the client to node 1. On a
+  single-socket host the pin / no-pin rows converge.
+- Both backends reach the same ~7 M plateau once each has its required ring config — uring leads by
+  5–8% at t=32–48, libaio by ~2% at t=64 — so pick either and tune the thread count (see
+  [Device Tuning](https://microsoft.github.io/garnet/docs/dev/device-tuning)).
+
+Reproduce with the checked-in generator (needs Release builds of `GarnetServer` + `Resp.benchmark`,
+`numactl`, and an NVMe / O_DIRECT mount):
+
+```bash
+DATA=/mnt/nvme/garnet benchmark/Resp.benchmark/scripts/nvme-raid0-matrix.sh
+```
+
+It sweeps both backends × pin/no-pin × threads, takes the median of `PASSES` (default 3) per cell, and
+prints the Markdown table above. Override `DATA`, `THREADS`, `PASSES`, `RUNTIME`, or set
+`CT` / `THROTTLE` / `URING_IOCTX` to run the explicitly tuned configuration instead of the defaults.
+Generator: [`scripts/nvme-raid0-matrix.sh`](scripts/nvme-raid0-matrix.sh).
+
 ## Offline variations
 
 ```bash