diff --git a/VX_types.toml b/VX_types.toml index ff3437215e..80ec9c17f6 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -848,6 +848,26 @@ VX_CSR_MPM_OCACHE_BANK_ST_H = 0xB8B VX_CSR_MPM_OCACHE_MSHR_ST = 0xB0C # MSHR stalls VX_CSR_MPM_OCACHE_MSHR_ST_H = 0xB8C +[csr_mpm_rtu] +# PERF: ray-tracing unit (per-socket; summed across the socket's RTU cores) +VX_CSR_MPM_RTU_RAYS = 0xB03 # rays traced (active lanes of the trace instructions) +VX_CSR_MPM_RTU_RAYS_H = 0xB83 +VX_CSR_MPM_RTU_HITS = 0xB04 # rays whose final result is a hit +VX_CSR_MPM_RTU_HITS_H = 0xB84 +VX_CSR_MPM_RTU_BOX_TESTS = 0xB05 # ray/box intersection tests +VX_CSR_MPM_RTU_BOX_TESTS_H = 0xB85 +VX_CSR_MPM_RTU_TRI_TESTS = 0xB06 # ray/triangle intersection tests +VX_CSR_MPM_RTU_TRI_TESTS_H = 0xB86 +VX_CSR_MPM_RTU_CALLBACKS = 0xB07 # any-hit and intersection candidates returned to the shader +VX_CSR_MPM_RTU_CALLBACKS_H = 0xB87 +VX_CSR_MPM_RTU_BUSY = 0xB08 # cycles with a ray in flight +VX_CSR_MPM_RTU_BUSY_H = 0xB88 +# PERF: ray-tracing cache +VX_CSR_MPM_RTCACHE_READS = 0xB09 # total reads +VX_CSR_MPM_RTCACHE_READS_H = 0xB89 +VX_CSR_MPM_RTCACHE_MISS_R = 0xB0A # read misses +VX_CSR_MPM_RTCACHE_MISS_R_H = 0xB8A + # XLEN is an environment-supplied build axis (configure exports it); the # [memmap]/[vm] exprs above reference it. Declared as a [[builtin]] so the # dependency is explicit and typed; defaults to 0 (→ 32-bit) when unset. diff --git a/ci/perfetto.py b/ci/perfetto.py index 9b554f47fa..cf1720dbd9 100755 --- a/ci/perfetto.py +++ b/ci/perfetto.py @@ -14,7 +14,7 @@ # limitations under the License. """ -vortex_perfetto.py +perfetto.py Convert Vortex GPU RTL or SimX log traces into a Perfetto-compatible trace (Chrome Trace JSON). @@ -30,13 +30,13 @@ Usage examples # SimX (default), compressed: - python3 vortex_perfetto.py run_simx.log -c -o vortex.perfetto.json.gz + python3 ci/perfetto.py run_simx.log -c -o vortex.perfetto.json.gz # RTL sim: - python3 vortex_perfetto.py run.log -t rtlsim -c -o vortex.perfetto.json.gz + python3 ci/perfetto.py run.log -t rtlsim -c -o vortex.perfetto.json.gz # Limit export to a cycle window: - python3 vortex_perfetto.py run.log -t rtlsim --cycle-min 10000 --cycle-max 20000 -c -o win.json.gz + python3 ci/perfetto.py run.log -t rtlsim --cycle-min 10000 --cycle-max 20000 -c -o win.json.gz Open in Perfetto UI - Web UI: https://ui.perfetto.dev (load the .json or .json.gz) diff --git a/ci/test_runner.py b/ci/test_runner.py index 2cf559d5f1..5feae9f5d6 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -16,6 +16,7 @@ instructions must match exactly (both are deterministic ISA-level executions, so any delta is functional divergence) and cycles must agree within the case's `tolerance` (SimX is the timing model of the RTL, not just a functional oracle). +A case with `parity_counters` also compares that MPM class's counters. """ import os @@ -32,7 +33,7 @@ def _run_one(case, xlen, driver): - """One parity leg: run under `driver`, return (instrs, cycles).""" + """One parity leg: run under `driver`, return (instrs, cycles, output).""" argv, env = case.run_command(xlen, driver=driver) rc, out = tc.execute_capture(argv, env) assert rc == 0, "{} [{}] failed (exit {}): {}".format( @@ -41,12 +42,59 @@ def _run_one(case, xlen, driver): assert perf, "{} [{}]: no 'PERF: instrs=…, cycles=…' summary in output " \ "(app must call vx_device_dump_perf)".format(case.id, driver) instrs, cycles = perf[-1] - return int(instrs), int(cycles) + return int(instrs), int(cycles), out + + +def parse_counters(out, prefix): + """The raw `name=value` counters of an MPM class's device-total line + (`PERF: : ...`; per-unit lines carry a `coreN:` prefix and do not + match). Derived metrics in parentheses are dropped.""" + lines = re.findall(r"^PERF: {}: (.*)$".format(re.escape(prefix)), out, re.M) + if not lines: + return None + raw = re.sub(r"\([^)]*\)", "", lines[-1].split(";")[0]) + return {k: int(v) for k, v in re.findall(r"(\w+)=(\d+)", raw)} + + +def rtu_invariants(c, cycles): + """Violations of the RTU counters' definitions (empty when consistent).""" + errs = [] + if c["hits"] > c["rays"]: + errs.append("hits {} > rays {}".format(c["hits"], c["rays"])) + if c["busy"] > cycles: + errs.append("busy {} > cycles {}".format(c["busy"], cycles)) + return errs + + +def _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles): + pc = case.parity_counters + prefix = pc["prefix"] + sc = parse_counters(simx_out, prefix) + rc = parse_counters(rtl_out, prefix) + assert sc and rc, "{}: no 'PERF: {}: ...' counter line (simx={}, rtlsim={})".format( + case.id, prefix, sc is not None, rc is not None) + if prefix == "rtu": + for drv, c, cyc in (("simx", sc, simx_cycles), ("rtlsim", rc, rtl_cycles)): + errs = rtu_invariants(c, cyc) + assert not errs, "{} [{}]: RTU counter invariants violated: {}".format( + case.id, drv, "; ".join(errs)) + fails = [] + for name in pc.get("exact", []): + print("PARITY: {}: {} simx={} rtlsim={}".format(case.id, name, sc.get(name), rc.get(name))) + if name not in sc or sc[name] != rc.get(name): + fails.append("{} simx={} rtlsim={}".format(name, sc.get(name), rc.get(name))) + for name in pc.get("tolerance", []): + s, r = sc.get(name), rc.get(name) + gap = abs(r - s) / float(max(r, 1)) if (s is not None and r is not None) else 1.0 + print("PARITY: {}: {} simx={} rtlsim={} gap={:.2%}".format(case.id, name, s, r, gap)) + if gap > case.tolerance: + fails.append("{} simx={} rtlsim={} gap={:.2%}".format(name, s, r, gap)) + assert not fails, "{}: {} counter mismatch: {}".format(case.id, prefix, "; ".join(fails)) def _model_parity(case, xlen): - simx_instrs, simx_cycles = _run_one(case, xlen, "simx") - rtl_instrs, rtl_cycles = _run_one(case, xlen, "rtlsim") + simx_instrs, simx_cycles, simx_out = _run_one(case, xlen, "simx") + rtl_instrs, rtl_cycles, rtl_out = _run_one(case, xlen, "rtlsim") gap = abs(rtl_cycles - simx_cycles) / float(rtl_cycles) print("PARITY: {}: instrs simx={} rtlsim={}, cycles simx={} rtlsim={}, " "gap={:.2%} (tolerance {:.0%})".format( @@ -58,11 +106,13 @@ def _model_parity(case, xlen): assert gap <= case.tolerance, \ "{}: cycle gap {:.2%} exceeds tolerance {:.0%} (simx={}, rtlsim={})".format( case.id, gap, case.tolerance, simx_cycles, rtl_cycles) + if case.parity_counters: + _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles) def _perf_gate(case, xlen, update): """Run the benchmark on rtlsim; record or compare its cycles vs baseline.""" - instrs, cycles = _run_one(case, xlen, "rtlsim") + instrs, cycles, _ = _run_one(case, xlen, "rtlsim") if update: pb.record(case, xlen, cycles, instrs) print("PERF-BASELINE: {} xlen={} record cycles={} instrs={}".format( diff --git a/ci/testcase.py b/ci/testcase.py index 0e8da16c65..c9845ca45a 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -108,6 +108,11 @@ def __init__(self, category, entry, driver, defaults): self.xlens = [32] self.tolerance = float(entry.get("tolerance", defaults.get("tolerance", DEFAULT_PARITY_TOLERANCE))) + # model_parity can also compare an MPM class's counters, read from the + # runtime's `PERF: : name=value, ...` device-total line (the run + # must pass the class with --perf=N): `exact` counters must match, and + # `tolerance` counters must agree within the case tolerance. + self.parity_counters = dict(entry.get("parity_counters", {})) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) # make-run / script fields self.dir = entry.get("dir", "") @@ -585,6 +590,18 @@ def cmd_lint(args): "(the runner drives simx and rtlsim)".format(c.id)) if not (0.0 < c.tolerance < 1.0): errors.append("{}: tolerance must be in (0, 1)".format(c.id)) + if c.parity_counters: + pc = c.parity_counters + if c.check != "model_parity": + errors.append("{}: parity_counters needs check: model_parity".format(c.id)) + if not isinstance(pc.get("prefix"), str) or not pc.get("prefix"): + errors.append("{}: parity_counters needs a 'prefix'".format(c.id)) + if set(pc) - {"prefix", "exact", "tolerance"}: + errors.append("{}: parity_counters keys are prefix/exact/tolerance".format(c.id)) + if not (pc.get("exact") or pc.get("tolerance")): + errors.append("{}: parity_counters compares nothing".format(c.id)) + if "--perf=" not in c.flags: + errors.append("{}: parity_counters needs --perf= in flags".format(c.id)) if errors: for e in errors: sys.stderr.write("LINT ERROR: " + e + "\n") diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index 734ef894cf..66c1e171db 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -85,6 +85,12 @@ tests: via: blackbox app: rt_raycast configs: -DVX_CFG_EXT_RTU_ENABLE + # The RTU counters too: the traversal work both models must do identically. + # busy is a timing counter and is left out. + flags: --perf=15 + parity_counters: + prefix: rtu + exact: [rays, hits, box_tests, tri_tests, callbacks] # Perf-regression twin of the model_parity-rt_raycast benchmark (same run, cycles-vs-baseline). - id: perf_gate-rt_raycast diff --git a/docs/designs/dxa_async_copy_multicast.md b/docs/designs/dxa_async_copy_multicast.md index 470a813472..fcb0c831eb 100644 --- a/docs/designs/dxa_async_copy_multicast.md +++ b/docs/designs/dxa_async_copy_multicast.md @@ -58,10 +58,10 @@ There is no software→global path; DXA is GMEM-read / LMEM-write only. `VX_CFG_NUM_DXA_UNITS = max(1, ceil(NUM_CORES/8))`, `VX_CFG_DXA_QUEUE_SIZE = 16`, `VX_CFG_DXA_MAX_INFLIGHT = 8`, `VX_DCR_DXA_DESC_COUNT = 16`. -- **Perf CSRs** `[csr_mpm_dxa]` 0xB03–0xB07 (transfers, gmem_reads, - gmem_dedup, lmem_writes, gmem_latency), - `VX_DCR_MPM_CLASS_DXA = 6` - ([`VX_types.toml:557-568`](../../VX_types.toml#L557)). +- **Perf CSRs** `[csr_mpm_dxa]` 0xB03–0xB08 (transfers, gmem_reads, + gmem_dedup, lmem_writes, gmem_latency, noslot_stalls), + `VX_DCR_MPM_CLASS_DXA = 16` + ([`VX_types.toml:767-780`](../../VX_types.toml#L767)). - **Cluster dispatch** (multicast prerequisite): `VX_CSR_CTA_CLUSTER_SIZE = 0xCE0`, `VX_DCR_KMU_CLUSTER_DIM_{X,Y,Z}` diff --git a/docs/designs/graphics_hardware_stack.md b/docs/designs/graphics_hardware_stack.md index 2b90dcd737..43132045a6 100644 --- a/docs/designs/graphics_hardware_stack.md +++ b/docs/designs/graphics_hardware_stack.md @@ -104,7 +104,8 @@ been **retired** in favor of this push/launch path (§4). rides the aperture address. DCRs are broadcast to all cluster instances; each raster instance self-selects its tile stripe by `INSTANCE_IDX`. -- **Perf** MPM classes RASTER=12, TEX=13, OM=14. +- **Perf** MPM classes RASTER=12, TEX=13, OM=14, RTU=15 (ray tracing, see + [ray_tracing_architecture.md §8.1](ray_tracing_architecture.md#81-performance-counters-mpm-class-15)). ### 2.1 The graphics register window (FF↔SIMT handoff) diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index 07b57b3318..24a7a3496d 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -467,10 +467,49 @@ oracle**: SimX additionally models **per-warp multi-trace concurrency** (several traces in flight from one warp), which the RTL core does not implement — SimX remains the -oracle for that. Perf: MPM class `RTU` plus the SimX `PerfStats` (context/slot -occupancy, front-end busy ticks, box/tri test counts); the RTL core keeps its own -per-slot occupancy counters (busy / write / callback / fill and the two idle -causes: idle-with-a-ray vs starved) under `DBG_RTU_OCC`. +oracle for that. Its exploration counters (context occupancy, fetch merging, +walker cycle accounting) stay in `PerfStats` and the `VX_RTU_STATS=1` teardown +dump; the architectural set below is what both models expose. + +### 8.1 Performance counters (MPM class 15) + +`--perf=15` (`VORTEX_PROFILING=15`, or `vx_device_mpm_query` with +`VX_DCR_MPM_CLASS_RTU`) reads the RTU counters on every driver. They are +**socket-scoped**: a socket sums its RTU cores and hands the result to each of its +cores' CSR file, so every core of a socket reads the same values. The runtime +queries one core per socket, prints a per-socket line when there is more than one +socket, and always prints the device total. + +The set is kept to quantities every BVH ray-tracing unit has, whatever its +internal organization, so the counters stay meaningful as the RTU evolves. +Microarchitecture-specific measures (slots, short-stack restarts, front-end steps) +are left to `PerfStats`; the class's CSR window (B03–B1F) leaves room for +future counters. + +| CSR | Counter | Definition | +|---|---|---| +| B03 | `RTU_RAYS` | rays traced: the active lanes of the trace instructions | +| B04 | `RTU_HITS` | rays whose final result is a hit (misses = rays − hits) | +| B05 | `RTU_BOX_TESTS` | ray/box intersection tests (node children and procedural boxes) | +| B06 | `RTU_TRI_TESTS` | ray/triangle intersection tests | +| B07 | `RTU_CALLBACKS` | any-hit and intersection candidates returned to the shader (one per ray per candidate) | +| B08 | `RTU_BUSY` | cycles with at least one ray in flight | +| B09 | `RTCACHE_READS` | RTU memory requests (RTCache reads) | +| B0A | `RTCACHE_MISS_R` | RTCache read misses | + +Box and triangle tests include the re-descents of restarted walks: SimX runs the +same `VX_CFG_RTU_STACK_DEPTH` short stack and trail restart as the RTL, so both +models do the same intersection work. Invariants: `RTU_HITS ≤ RTU_RAYS`, +`RTU_BUSY ≤ cycles`. + +The `model_parity-rt_raycast` case requires rays, hits, box tests, triangle tests +and callbacks to match SimX exactly. `RTU_BUSY` and the RTCache counters are +timing and memory-system measures and are not gated: SimX keeps a ray's fetched +lines where the RTL refetches per structure access. + +The runtime also derives the hit rate, box and triangle tests per ray, memory +requests per ray, RTU utilization and the RTCache hit rate. In RTL the counters +exist only in `PERF_ENABLE` builds. --- diff --git a/docs/perfetto_analysis.md b/docs/perfetto_analysis.md index 843b8b4dec..e0217b81b6 100644 --- a/docs/perfetto_analysis.md +++ b/docs/perfetto_analysis.md @@ -1,6 +1,6 @@ # Vortex Perfetto Trace: Performance Analysis Guide -This guide explains how to **generate** and **interpret** a Vortex GPU trace that has been converted into **Perfetto-compatible Chrome Trace JSON** using `vortex_perfetto.py`. +This guide explains how to **generate** and **interpret** a Vortex GPU trace that has been converted into **Perfetto-compatible Chrome Trace JSON** using `ci/perfetto.py`. It is written for people who are: - New to Perfetto @@ -33,13 +33,13 @@ This guide focuses on **Vortex GPU 1**. #### SimX (default) ```bash -python3 vortex_perfetto.py run_simx.log -t simx -c -o vortex.perfetto.json.gz +python3 ci/perfetto.py run_simx.log -t simx -c -o vortex.perfetto.json.gz ``` #### RTL simulation ```bash -python3 vortex_perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.gz +python3 ci/perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.gz ``` ### Common options @@ -47,7 +47,7 @@ python3 vortex_perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.g - **Limit export to a cycle/tick window** (highly recommended for huge logs): ```bash -python3 vortex_perfetto.py run_rtlsim.log -t rtlsim \ +python3 ci/perfetto.py run_rtlsim.log -t rtlsim \ --cycle-min 10000 --cycle-max 20000 -c -o window.json.gz ``` diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 53193421c1..88a1b908a4 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1316,6 +1316,17 @@ package VX_gpu_pkg; } om_perf_t; `endif +`ifdef VX_CFG_EXT_RTU_ENABLE + typedef struct packed { + logic [PERF_CTR_BITS-1:0] rays; + logic [PERF_CTR_BITS-1:0] hits; + logic [PERF_CTR_BITS-1:0] box_tests; + logic [PERF_CTR_BITS-1:0] tri_tests; + logic [PERF_CTR_BITS-1:0] callbacks; + logic [PERF_CTR_BITS-1:0] busy; + } rtu_perf_t; +`endif + typedef struct packed { logic [PERF_CTR_BITS-1:0] idles; logic [PERF_CTR_BITS-1:0] active_warps; @@ -1361,6 +1372,10 @@ package VX_gpu_pkg; om_perf_t om; cache_perf_t ocache; `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + rtu_perf_t rtu; + cache_perf_t rtcache; + `endif } sysmem_perf_t; typedef struct packed { diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index c14be8a0c4..af9b3258aa 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -104,6 +104,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE tex_perf_t socket_tex_perf; cache_perf_t socket_tcache_perf; +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + rtu_perf_t socket_rtu_perf; + cache_perf_t socket_rtcache_perf; `endif sysmem_perf_t sysmem_perf_tmp; always @(*) begin @@ -117,6 +121,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; sysmem_perf_tmp.tex = socket_tex_perf; sysmem_perf_tmp.tcache = socket_tcache_perf; `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + sysmem_perf_tmp.rtu = socket_rtu_perf; + sysmem_perf_tmp.rtcache = socket_rtcache_perf; + `endif end `endif @@ -575,6 +583,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .TAG_WIDTH (RTCACHE_TAG_WIDTH) ) rtcache_bus_if [`VX_CFG_NUM_RTU_CORES * RTCACHE_NUM_REQS] (); +`ifdef PERF_ENABLE + rtu_perf_t [`VX_CFG_NUM_RTU_CORES-1:0] per_core_rtu_perf; +`endif + for (genvar i = 0; i < `VX_CFG_NUM_RTU_CORES; ++i) begin : g_rtu_core VX_rtu_core #( .INSTANCE_ID (`SFORMATF(("%s-rtu%0d", INSTANCE_ID, i))), @@ -586,11 +598,30 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ) rtu_core ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + .perf_rtu (per_core_rtu_perf[i]), + `endif .rtu_bus_if (rtu_bus_if[i]), .cache_bus_if (rtcache_bus_if[i * RTCACHE_NUM_REQS]) ); end +`ifdef PERF_ENABLE + // Every rtu_perf_t field is one PERF_CTR_BITS counter: sum them field-wise. + localparam RTU_PERF_FIELDS = $bits(rtu_perf_t) / PERF_CTR_BITS; + wire [`VX_CFG_NUM_RTU_CORES-1:0][RTU_PERF_FIELDS-1:0][PERF_CTR_BITS-1:0] rtu_perf_w = per_core_rtu_perf; + reg [RTU_PERF_FIELDS-1:0][PERF_CTR_BITS-1:0] rtu_perf_sum; + always @(*) begin + rtu_perf_sum = '0; + for (integer i = 0; i < `VX_CFG_NUM_RTU_CORES; ++i) begin + for (integer f = 0; f < RTU_PERF_FIELDS; ++f) begin + rtu_perf_sum[f] = rtu_perf_sum[f] + rtu_perf_w[i][f]; + end + end + end + assign socket_rtu_perf = rtu_perf_sum; +`endif + VX_mem_bus_if #( .DATA_SIZE (RTCACHE_LINE_SIZE), .TAG_WIDTH (RTCACHE_MEM_TAG_WIDTH) @@ -647,7 +678,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), `ifdef PERF_ENABLE - `UNUSED_PIN (cache_perf), + .cache_perf (socket_rtcache_perf), `endif .core_bus_if (rtcache_flushable_bus_if), .mem_bus_if (rtcache_mem_bus_tmp_if) diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index f09c17bb7e..a47d6a86be 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -573,6 +573,21 @@ import VX_fpu_pkg::*; default:; endcase end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `VX_DCR_MPM_CLASS_RTU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); + `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); + `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); + `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); + default:; + endcase + end `endif default:; endcase diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index 3f018a13e0..932bcaa8a5 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -53,6 +53,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + output rtu_perf_t perf_rtu, +`endif + // window bus (this core is the master) VX_rtu_bus_if.slave rtu_bus_if, @@ -196,6 +200,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( `UNUSED_VAR (sch_busy) +`ifdef PERF_ENABLE + rtu_sched_events_t sch_events; +`endif + VX_rtu_scheduler #( .INSTANCE_ID (INSTANCE_ID), .NUM_SLOTS (NUM_SLOTS), @@ -203,6 +211,9 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ) scheduler ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + .perf_events (sch_events), + `endif .slot_start_valid (ss_valid), .slot_start_slot (ss_slot), .slot_start_mask (ss_mask), @@ -786,45 +797,65 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( end end -// ── RTU occupancy counters ──────────────────────────────────────────────── -`ifdef DBG_RTU_OCC - longint unsigned occ_total, occ_busy, occ_write, occ_cb, occ_fill; - longint unsigned occ_idle_ray_waiting, occ_idle_starved, occ_traces; +`ifdef PERF_ENABLE + localparam LANE_CNT_W = `CLOG2(NUM_LANES + 1); + + // A record's lanes are counted once, when its status word (the last) goes out. + wire perf_rec_done = win_fire && wr_status; + wire perf_term = perf_rec_done && !is_cand[ws]; + wire perf_cand = perf_rec_done && is_cand[ws]; + + wire [NUM_LANES-1:0] ws_hit = sch_hit[32'(ws)*NUM_LANES +: NUM_LANES]; + + // Shader callbacks: any-hit and intersection candidates returned to the warp. + wire [NUM_LANES-1:0] ws_cb; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_perf_cb + assign ws_cb[l] = wr_cb_mask[l] + && ((sch_cbtype[wctx[l]] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_ANYHIT)) + || (sch_cbtype[wctx[l]] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_PROC))); + end + + wire [NUM_LANES-1:0] perf_ray_lanes = ss_valid ? ss_mask : '0; + wire [NUM_LANES-1:0] perf_hit_lanes = perf_term ? (req_mask[ws] & ws_hit) : '0; + wire [NUM_LANES-1:0] perf_cb_lanes = perf_cand ? ws_cb : '0; + + wire [LANE_CNT_W-1:0] perf_rays_cnt, perf_hits_cnt, perf_cb_cnt; + `POP_COUNT(perf_rays_cnt, perf_ray_lanes); + `POP_COUNT(perf_hits_cnt, perf_hit_lanes); + `POP_COUNT(perf_cb_cnt, perf_cb_lanes); + + // The per-cycle increments are registered so the counter adders stay off + // the scheduler's and the record walk's timing paths. + rtu_sched_events_t sch_events_r; + reg busy_r; + reg [LANE_CNT_W-1:0] rays_r, hits_r, cb_r; + rtu_perf_t perf_rtu_r; + always @(posedge clk) begin if (reset) begin - occ_total <= 0; occ_busy <= 0; occ_write <= 0; occ_cb <= 0; - occ_fill <= 0; occ_idle_ray_waiting <= 0; occ_idle_starved <= 0; - occ_traces <= 0; + sch_events_r <= '0; + busy_r <= 1'b0; + rays_r <= '0; + hits_r <= '0; + cb_r <= '0; + perf_rtu_r <= '0; end else begin - occ_total <= occ_total + 1; - if (ss_valid) begin - occ_traces <= occ_traces + 1; - end - for (integer s = 0; s < NUM_SLOTS; s = s + 1) begin - case (tstate[s]) - T_BUSY: occ_busy <= occ_busy + 1; - T_WRITE: occ_write <= occ_write + 1; - T_FILL: occ_fill <= occ_fill + 1; - T_CBWAIT, T_CBATTR, T_RESUME, T_RWAIT: occ_cb <= occ_cb + 1; - T_IDLE: begin - if ((| stg_ready) || (fstate == F_RUN)) begin - occ_idle_ray_waiting <= occ_idle_ray_waiting + 1; - end else begin - occ_idle_starved <= occ_idle_starved + 1; - end - end - default:; - endcase - end - end - end - always @(posedge clk) begin - if (!reset && (occ_total % 40000 == 39999)) begin - $display("RTU-OCC @%0d: traces=%0d | busy=%0d write=%0d cb=%0d fill=%0d | IDLE_ray_waiting=%0d IDLE_STARVED=%0d", - occ_total, occ_traces, occ_busy, occ_write, occ_cb, occ_fill, - occ_idle_ray_waiting, occ_idle_starved); + sch_events_r <= sch_events; + busy_r <= ~(& slot_free); + rays_r <= perf_rays_cnt; + hits_r <= perf_hits_cnt; + cb_r <= perf_cb_cnt; + + perf_rtu_r.rays <= perf_rtu_r.rays + PERF_CTR_BITS'(rays_r); + perf_rtu_r.hits <= perf_rtu_r.hits + PERF_CTR_BITS'(hits_r); + perf_rtu_r.box_tests <= perf_rtu_r.box_tests + PERF_CTR_BITS'(sch_events_r.box_test); + perf_rtu_r.tri_tests <= perf_rtu_r.tri_tests + PERF_CTR_BITS'(sch_events_r.tri_test); + perf_rtu_r.callbacks <= perf_rtu_r.callbacks + PERF_CTR_BITS'(cb_r); + perf_rtu_r.busy <= perf_rtu_r.busy + PERF_CTR_BITS'(busy_r); end end + + assign perf_rtu = perf_rtu_r; `endif endmodule diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index 2f31dfe102..872e3df571 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -303,6 +303,13 @@ package VX_rtu_pkg; localparam RTU_WS_WORDS = 32; // rows per slot (power of two for addressing) localparam RTU_WS_WORD_BITS = `CLOG2(RTU_WS_WORDS); + // Per-cycle traversal events the scheduler reports for the performance + // counters: strobes only, the counting lives in the core. + typedef struct packed { + logic box_test; // a ray/AABB test issued + logic tri_test; // a ray/triangle test issued + } rtu_sched_events_t; + `IGNORE_UNUSED_END endpackage diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 46f1676251..dab898a532 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -61,6 +61,10 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + output rtu_sched_events_t perf_events, +`endif + // ── slot launch (from the core's fill engine) ───────────────────── // slot_start arms the slot (mask + warp-uniform ray scalars) before any of // its rays land; each lane's ray then arrives on ray_wr, which is also that @@ -1908,6 +1912,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( ("%t: rtu walk deeper than %0d levels", $time, (1 << LVLW) - 1)) assign path_wlvl = path_wlvl_r; assign path_wdata = path_wdata_r; + +`ifdef PERF_ENABLE + assign perf_events.box_test = box_feed || box_feed_raw; + assign perf_events.tri_test = tri_feed; +`endif assign mem_req_valid = x_valid && mem_issue; assign mem_req_addr = structaddr_q + (ADDRW'(mem_fslot) << RTU_LINE_SEL_BITS); assign mem_req_tag = sel_q; diff --git a/hw/unittest/rtu_core/VX_rtu_core_top.sv b/hw/unittest/rtu_core/VX_rtu_core_top.sv index f4405346dd..f023e8f316 100644 --- a/hw/unittest/rtu_core/VX_rtu_core_top.sv +++ b/hw/unittest/rtu_core/VX_rtu_core_top.sv @@ -123,6 +123,9 @@ module VX_rtu_core_top import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ) rtu_core ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rtu), + `endif .rtu_bus_if (rtu_bus_if), .cache_bus_if (cache_bus_if) ); diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index c000ef2fa5..9291bf8a86 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -263,15 +263,15 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { #endif #ifdef VX_CFG_EXT_TEX_ENABLE case VX_DCR_MPM_CLASS_TEX: { - auto cluster_perf = core_->socket()->cluster()->perf_stats(); + auto socket_perf = core_->socket()->perf_stats(); switch (addr) { - CSR_READ_64(VX_CSR_MPM_TEX_READS, cluster_perf.tex.mem_reads); - CSR_READ_64(VX_CSR_MPM_TEX_LAT, cluster_perf.tex.mem_latency); - CSR_READ_64(VX_CSR_MPM_TEX_ST, cluster_perf.tex.stall_cycles); - CSR_READ_64(VX_CSR_MPM_TCACHE_READS, cluster_perf.tcache.reads); - CSR_READ_64(VX_CSR_MPM_TCACHE_MISS_R, cluster_perf.tcache.read_misses); - CSR_READ_64(VX_CSR_MPM_TCACHE_BANK_ST,cluster_perf.tcache.bank_stalls); - CSR_READ_64(VX_CSR_MPM_TCACHE_MSHR_ST,cluster_perf.tcache.mshr_stalls); + CSR_READ_64(VX_CSR_MPM_TEX_READS, socket_perf.tex.mem_reads); + CSR_READ_64(VX_CSR_MPM_TEX_LAT, socket_perf.tex.mem_latency); + CSR_READ_64(VX_CSR_MPM_TEX_ST, socket_perf.tex.stall_cycles); + CSR_READ_64(VX_CSR_MPM_TCACHE_READS, socket_perf.tcache.reads); + CSR_READ_64(VX_CSR_MPM_TCACHE_MISS_R, socket_perf.tcache.read_misses); + CSR_READ_64(VX_CSR_MPM_TCACHE_BANK_ST,socket_perf.tcache.bank_stalls); + CSR_READ_64(VX_CSR_MPM_TCACHE_MSHR_ST,socket_perf.tcache.mshr_stalls); } } break; #endif @@ -305,6 +305,21 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { CSR_READ_64(VX_CSR_MPM_OCACHE_MSHR_ST,cluster_perf.ocache.mshr_stalls); } } break; + #endif + #ifdef VX_CFG_EXT_RTU_ENABLE + case VX_DCR_MPM_CLASS_RTU: { + auto socket_perf = core_->socket()->perf_stats(); + switch (addr) { + CSR_READ_64(VX_CSR_MPM_RTU_RAYS, socket_perf.rtu.rays); + CSR_READ_64(VX_CSR_MPM_RTU_HITS, socket_perf.rtu.rays_hit); + CSR_READ_64(VX_CSR_MPM_RTU_BOX_TESTS, socket_perf.rtu.bvh_box_tests); + CSR_READ_64(VX_CSR_MPM_RTU_TRI_TESTS, socket_perf.rtu.bvh_tri_tests); + CSR_READ_64(VX_CSR_MPM_RTU_CALLBACKS, (socket_perf.rtu.ahs_callbacks + socket_perf.rtu.is_callbacks)); + CSR_READ_64(VX_CSR_MPM_RTU_BUSY, socket_perf.rtu.rtu_busy_ticks); + CSR_READ_64(VX_CSR_MPM_RTCACHE_READS, socket_perf.rtcache.reads); + CSR_READ_64(VX_CSR_MPM_RTCACHE_MISS_R, socket_perf.rtcache.read_misses); + } + } break; #endif default: std::cerr << "Error: invalid MPM CLASS: value=" << perf_class << std::endl; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 5f3c357644..1af9074776 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -138,7 +138,7 @@ class DxaCore::Impl { void reset() { cycle_ = 0; queue_.clear(); - perf_stats_ = DxaCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& w : workers_) { w.state = WState::IDLE; w.work_list.clear(); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 8c4dcfc0a8..df1a19d7ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -688,7 +688,7 @@ class CacheBank : public SimObject { protected: void on_reset() { - perf_stats_ = Cache::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. pending_mshr_size_ = 0; pending_amo_probes_ = 0; pipe_fill_count_ = 0; diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index d2b6d60598..41b82e8575 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -99,7 +99,7 @@ class LocalMem::Impl { virtual ~Impl() {} void reset() { - perf_stats_ = PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index 7cc253575e..0edd322f21 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -88,11 +88,13 @@ void LocalMemSwitch::on_tick() { out_lmem_req.addrs.at(i) = in_req.addrs.at(i); out_lmem_req.data.at(i) = in_req.data.at(i); out_lmem_req.byteen.at(i) = in_req.byteen.at(i); + out_lmem_req.amo_cmp.at(i) = in_req.amo_cmp.at(i); } else { out_dc_req.mask.set(i); out_dc_req.addrs.at(i) = in_req.addrs.at(i); out_dc_req.data.at(i) = in_req.data.at(i); out_dc_req.byteen.at(i) = in_req.byteen.at(i); + out_dc_req.amo_cmp.at(i) = in_req.amo_cmp.at(i); } } } diff --git a/sim/simx/om/om_core.cpp b/sim/simx/om/om_core.cpp index ef8a911e8f..d7bf17aad6 100644 --- a/sim/simx/om/om_core.cpp +++ b/sim/simx/om/om_core.cpp @@ -128,7 +128,7 @@ class OmCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = OmCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& s : slots_) { s.in_use = false; s.state = State::ADDR; diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 52ad76ca1a..cc9116cfb3 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -331,9 +331,7 @@ void ProcessorImpl::forward_delegated_launch() { void ProcessorImpl::reset() { SimPlatform::instance().reset(); - perf_mem_reads_ = 0; - perf_mem_writes_ = 0; - perf_mem_latency_ = 0; + // The perf_mem_* counters survive: the MPM counters run from device reset. perf_mem_pending_reads_ = 0; is_cycle_initialized_ = false; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index c9c7c7692d..ebe59cf297 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -95,10 +95,10 @@ class ProcessorImpl { #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_; - uint64_t perf_mem_reads_; - uint64_t perf_mem_writes_; - uint64_t perf_mem_latency_; - uint64_t perf_mem_pending_reads_; + uint64_t perf_mem_reads_ = 0; + uint64_t perf_mem_writes_ = 0; + uint64_t perf_mem_latency_ = 0; + uint64_t perf_mem_pending_reads_ = 0; // Tracks whether cycle() has done its first-call init (reset + // kmu_->start()). reset() clears it so a back-to-back kernel launch // via cycle() re-dispatches the KMU. diff --git a/sim/simx/raster/raster_core.cpp b/sim/simx/raster/raster_core.cpp index 9b7b16cc22..4cb79ad943 100644 --- a/sim/simx/raster/raster_core.cpp +++ b/sim/simx/raster/raster_core.cpp @@ -152,7 +152,7 @@ class RasterCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = RasterCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. reset_load_state(); state_ = State::IDLE; // The per-launch reset precedes the delegated launch's frame kick, so both diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index b8f53f321e..3ad2f0a7e8 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -184,8 +184,6 @@ class ReformationEngine { port.send(rsp); warp_cb_inflight_[anchor_warp] = true; ++perf_.reformation_yields; - // cb_type is uniform within a batched yield (grouped by sbt_idx), so one - // sample names the per-yield-type counter. for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (!(cb_mask & (1u << t))) continue; switch (rsp.cb_type[t]) { @@ -195,7 +193,6 @@ class ReformationEngine { case VX_RT_CB_TYPE_MISS: ++perf_.miss_callbacks; break; default: break; } - break; } DT(3, "rtu-core reform cb_yield: warp=" << anchor_warp << ", sbt=" << anchor_sbt @@ -255,7 +252,7 @@ class RtuCore::Impl { const auto& p = perf_stats_; std::fprintf(stderr, "[rtu-stats] slots=%u ctx=%u merge=%u\n" - "[rtu-stats] rays_issued=%llu rays_hit=%llu rays_miss=%llu\n" + "[rtu-stats] traces=%llu rays=%llu rays_hit=%llu rays_miss=%llu\n" "[rtu-stats] mem_reads=%llu fetches_merged=%llu mshr_full=%llu\n" "[rtu-stats] bvh_nodes=%llu bvh_leaves=%llu box_tests=%llu tri_tests=%llu\n" "[rtu-stats] busy_ticks=%llu front_end_ticks=%llu mem_issue_ticks=%llu\n" @@ -263,7 +260,8 @@ class RtuCore::Impl { "[rtu-stats] cb_ahs=%llu cb_chs=%llu cb_miss=%llu cb_is=%llu reform=%llu\n", uint32_t(VX_CFG_RTU_NUM_SLOTS), uint32_t(VX_CFG_RTU_NUM_CTX), uint32_t(VX_CFG_RTU_MERGE_DEPTH), - (unsigned long long)p.rays_issued, + (unsigned long long)p.traces, + (unsigned long long)p.rays, (unsigned long long)p.rays_hit, (unsigned long long)p.rays_miss, (unsigned long long)p.mem_reads, @@ -291,7 +289,8 @@ class RtuCore::Impl { for (auto& cx : contexts_) cx = Context{}; mem_engine_.reset(); reform_.reset(); - perf_stats_ = RtuCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset, and every + // kernel launch re-enters reset(). last_signature_ = 0; rr_exec_ = 0; step_cycles_left_ = 0; @@ -359,7 +358,8 @@ class RtuCore::Impl { } if (raylog::enabled()) raylog::on_accept(this, idx, s.req); ch.pop(); - ++perf_stats_.rays_issued; + ++perf_stats_.traces; + perf_stats_.rays += __builtin_popcount(s.req.tmask_bits); DT(3, "rtu-core accept: tag=" << s.req.tag << ", slot=" << idx); rr_req_ = (cid + 1) % chs.size(); return; @@ -569,13 +569,12 @@ class RtuCore::Impl { + inst * kRtuImageStatesPerInst; // Behind the last test fed, the PE pipeline drains while the context is // PARKED — box and tri results both carry their context id, so waiting for - // one costs the shared front end nothing. A trail restart re-descends every - // subtree the short stack had to evict. + // one costs the shared front end nothing. A trail restart's re-descent is + // already in the node and box counts above. uint32_t lat = 0; if (box) lat = std::max(lat, BoxPe::pipe_depth()); if (tri) lat = std::max(lat, TriPe::pipe_depth()); lat += kRtuXformLatency * inst; - lat += VX_CFG_RTU_STACK_DEPTH * restart; cx.pe_lat = lat; perf_stats_.walker_cycles_total += cx.fsm_states * kRtuPhasesPerStep + cx.img_states * kRtuPhasesImageAdd + lat; @@ -723,8 +722,18 @@ class RtuCore::Impl { void sample_occupancy() { uint32_t slots_used = 0; + bool busy = false; for (uint32_t i = 0; i < pool_.size(); ++i) { - if (pool_.at(i).in_use) ++slots_used; + const Slot& s = pool_.at(i); + if (s.in_use) ++slots_used; + // A slot is reserved before its rays arrive and stays emitted until the + // warp consumes its record; it holds no trace in either state. + if (s.in_use && s.state != SlotState::RESERVED && s.state != SlotState::EMITTED) { + busy = true; + } + } + if (busy) { + ++perf_stats_.rtu_busy_ticks; } uint32_t bound = 0, active = 0; for (const auto& cx : contexts_) { @@ -738,7 +747,6 @@ class RtuCore::Impl { perf_stats_.slot_busy_ticks += slots_used; perf_stats_.ctx_bound_ticks += bound; perf_stats_.ctx_active_ticks += active; - if (slots_used != 0) ++perf_stats_.rtu_busy_ticks; } const RtuCore::PerfStats& perf_stats() const { return perf_stats_; } diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8e84ba11cb..cb72cd528a 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -542,7 +542,8 @@ struct QueueEntry { // ════════════════════════════════════════════════════════════════════ struct PerfStats { - uint64_t rays_issued = 0; + uint64_t traces = 0; // warp-level traces accepted + uint64_t rays = 0; // active lanes of those traces uint64_t rays_hit = 0; uint64_t rays_miss = 0; uint64_t mem_reads = 0; @@ -552,12 +553,10 @@ struct PerfStats { uint64_t bvh_instance_descents = 0; uint64_t bvh_box_tests = 0; uint64_t bvh_tri_tests = 0; - // Short-stack overflow events: pushes past the VX_CFG_RTU_STACK_DEPTH HW - // stack. SimX keeps an unbounded stack (never misses a hit); each overflow - // entry is one the HW must re-descend for via trail-based restart, charged - // in the cost model. + // Trail restarts of the VX_CFG_RTU_STACK_DEPTH short stack. The node, box and + // descent counts above include the restarted walks' re-descents. uint64_t bvh_stack_restarts = 0; - // Callback-pipeline counters. + // Callback-pipeline counters: candidate lanes returned to the warp, by type. uint64_t ahs_callbacks = 0; uint64_t chs_callbacks = 0; uint64_t miss_callbacks = 0; @@ -586,14 +585,15 @@ struct PerfStats { uint64_t ctx_bound_ticks = 0; uint64_t ctx_active_ticks = 0; uint64_t slot_busy_ticks = 0; - uint64_t rtu_busy_ticks = 0; // ticks with any slot in use + uint64_t rtu_busy_ticks = 0; // ticks with any slot holding a trace // SIMD-PE cycle accounting: pipeline cycles charged across every context's // traversal, and ticks where at least one context was draining them. uint64_t walker_cycles_total = 0; uint64_t walker_busy_ticks = 0; PerfStats& operator+=(const PerfStats& rhs) { - rays_issued += rhs.rays_issued; + traces += rhs.traces; + rays += rhs.rays; rays_hit += rhs.rays_hit; rays_miss += rhs.rays_miss; mem_reads += rhs.mem_reads; diff --git a/sim/simx/rtu/rtu_walker.cpp b/sim/simx/rtu/rtu_walker.cpp index 3164e9f9d7..eaedcc9c8e 100644 --- a/sim/simx/rtu/rtu_walker.cpp +++ b/sim/simx/rtu/rtu_walker.cpp @@ -14,6 +14,7 @@ #include "rtu_walker.h" #include +#include #include #include #include @@ -124,21 +125,55 @@ inline uint32_t hit_facing_bit(bool back_facing, uint32_t inst_flags) { return back_facing ? VX_RT_HIT_BACK_FACING : 0u; } +// The hardware's per-ray traversal stack: VX_CFG_RTU_STACK_DEPTH entries shared +// by the TLAS and every BLAS below it. A walk visits the tree depth first, +// nearer child first, so every node has a rank path (its child rank at each +// level, an instance's index within its leaf) and the walk proceeds in +// ascending rank-path order. A child that does not fit on the stack is dropped; +// the walk then only descends, so when it would next pop it restarts from the +// root instead, following the rank path to the nearest dropped child: every +// node before it has been visited. A tightened best_t only culls a suffix of a +// node's ordered children, so the recorded ranks stay valid across restarts. +struct ShortStack { + static constexpr uint32_t kLevels = 64; + struct Entry { + uint32_t lvl; + uint32_t rank; + uint32_t off; + }; + std::vector entries; + std::array path{}; // rank taken at each level + bool ovf = false; // a child was dropped since the walk (re)started + uint32_t nd_lvl = 0; // the nearest dropped child + uint32_t nd_rank = 0; + bool follow = false; // restarting: descend along the rank path + uint32_t trl_lvl = 0; // ... down to this dropped child + uint32_t trl_rank = 0; + bool restart = false; // the walk must unwind and start over + uint64_t visits = 0; + + void set_path(uint32_t lvl, uint32_t rank) { + if (lvl < kLevels) path[lvl] = rank; + } + uint32_t get_path(uint32_t lvl) const { + return (lvl < kLevels) ? path[lvl] : 0; + } +}; + // Depth-first walker for one BVH sub-tree under the supplied (object-space) -// ray. Recurses on LeafInst so each instance's BLAS gets walked with its -// transformed ray. ctx accumulates hits/yields across the whole call tree. -// -// Returns as soon as sv.miss is raised: the caller discards the partial state -// and replays the walk once the missing line has arrived. +// ray, rooted at level root_lvl. Recurses on LeafInst so each instance's BLAS +// gets walked with its transformed ray on the same short stack. ctx accumulates +// hits/yields across the whole call tree. // -// The functional traversal stack is unbounded so the oracle never misses a hit; -// the HW short-stack (VX_CFG_RTU_STACK_DEPTH) overflow is counted as a -// trail-based RESTART and charged in the cost model. +// Returns as soon as sv.miss is raised (the caller discards the partial state +// and replays the walk once the missing line has arrived) or ss.restart is set +// (the caller restarts the walk from the scene root). void walk_bvh4_subtree(SceneView& sv, const float ro[3], const float rd[3], - uint32_t root_off, uint32_t instance_id, - uint32_t custom_id, uint32_t inst_flags, - WalkCtx& ctx, PerfStats& perf) { + uint32_t root_off, uint32_t root_lvl, + uint32_t instance_id, uint32_t custom_id, + uint32_t inst_flags, WalkCtx& ctx, ShortStack& ss, + PerfStats& perf) { // The ray setup's reciprocals, once per (object-space) ray. const float inv[3] = { ray_recip(rd[0]), ray_recip(rd[1]), ray_recip(rd[2]) }; @@ -263,12 +298,14 @@ void walk_bvh4_subtree(SceneView& sv, } }; - auto visit_leaf_inst = [&](uint32_t leaf_off, uint32_t count) { + auto visit_leaf_inst = [&](uint32_t leaf_off, uint32_t count, uint32_t lvl) { uint8_t hdr_buf[kVxBvhLeafHeaderBytes]; read_scene_bytes(sv, leaf_off, sizeof(hdr_buf), hdr_buf); if (sv.miss) return; uint32_t insts_off = leaf_off + kVxBvhLeafHeaderBytes; - for (uint32_t i = 0; i < count; ++i) { + // A restart following its path resumes at the instance it recorded. + const uint32_t first = ss.follow ? ss.get_path(lvl + 1) : 0; + for (uint32_t i = first; i < count; ++i) { uint8_t inst_buf[kVxBvhInstanceStride]; read_scene_bytes(sv, insts_off + i * kVxBvhInstanceStride, kVxBvhInstanceStride, inst_buf); @@ -278,41 +315,41 @@ void walk_bvh4_subtree(SceneView& sv, // Vulkan instanceCullMask: skip the instance entirely if its mask byte // and the ray's cull_mask have no bits in common. Both default to 0xff in // the no-culling path. - if ((inst->cull_mask & ctx.ray_cull_mask & 0xffu) == 0) continue; + if ((inst->cull_mask & ctx.ray_cull_mask & 0xffu) == 0) { + ss.follow = false; + continue; + } // VkGeometryInstanceFlagBits packed into cull_mask bits 15..8. uint32_t inst_flags2 = (inst->cull_mask >> kRtuInstanceFlagsShift) & kRtuInstanceFlagsMask; float obj_ro[3], obj_rd[3]; world_to_object_ray(inst->xform, ro, rd, obj_ro, obj_rd); ++perf.bvh_instance_descents; + // The BLAS root sits at the instance level, ranked by its index. + ss.set_path(lvl + 1, i); walk_bvh4_subtree(sv, obj_ro, obj_rd, - inst->blas_root_byte_offset, + inst->blas_root_byte_offset, lvl + 1, inst->instance_id, inst->custom_id, inst_flags2, - ctx, perf); - if (sv.miss) return; + ctx, ss, perf); + if (sv.miss || ss.restart || ctx.terminated) return; + ss.follow = false; } }; - // SimX is the correctness oracle: keep an UNBOUNDED traversal stack so deep - // sub-trees are never dropped. The HW short-stack is only - // VX_CFG_RTU_STACK_DEPTH deep and trail-restarts to re-find subtrees it had - // to evict — it visits the same leaves, just at extra cost. - std::vector stack; - stack.reserve(VX_CFG_RTU_STACK_DEPTH); - uint32_t current = root_off; - bool have_current = true; - // Safety backstop: a malformed/cyclic acceleration structure (a child offset - // pointing back at an ancestor) would otherwise descend forever with the - // unbounded stack. The ceiling is far above any well-formed scene's node - // count, so it never truncates a legitimate walk. + // pointing back at an ancestor) would otherwise descend forever. The ceiling + // is far above any well-formed scene's node count, so it never truncates a + // legitimate walk. constexpr uint64_t kMaxNodeVisits = 1ull << 22; - uint64_t node_visits = 0; - while (have_current) { - if (ctx.terminated) break; - if (++node_visits > kMaxNodeVisits) break; + // The stack depth this sub-tree started from: popping back to it ends a BLAS. + const size_t stack_floor = ss.entries.size(); + uint32_t current = root_off; + uint32_t lvl = root_lvl; + + while (!ctx.terminated) { + if (++ss.visits > kMaxNodeVisits) return; uint8_t kind_buf[4]; read_scene_bytes(sv, current, sizeof(kind_buf), kind_buf); if (sv.miss) return; @@ -329,8 +366,8 @@ void walk_bvh4_subtree(SceneView& sv, } } else if (kind == kVxBvhKindLeafInst) { ++perf.bvh_leaves_fetched; - visit_leaf_inst(current, count); - if (sv.miss) return; + visit_leaf_inst(current, count, lvl); + if (sv.miss || ss.restart) return; } else if (kind == kVxBvhKindLeafProc) { ++perf.bvh_leaves_fetched; // SKIP_AABBS: symmetric gate with SKIP_TRIANGLES. Otherwise ray-test each @@ -388,27 +425,53 @@ void walk_bvh4_subtree(SceneView& sv, } hits[j] = h; } - if (hit_count > 0) { - for (uint32_t i = hit_count; i-- > 1; ) { - // Each push past the HW short-stack depth is one subtree the HW would - // have to re-descend for via restart — count it for the cost model, - // but keep it (unbounded) so the functional walk never misses a hit. - if (stack.size() >= VX_CFG_RTU_STACK_DEPTH) - ++perf.bvh_stack_restarts; - stack.push_back(hits[i].offset); + + // A restart descends into the child its path names; nearer children + // were visited before. Culled by now, that child and every farther one + // are done. + uint32_t dsc = 0; + bool descend = (hit_count != 0); + if (ss.follow) { + dsc = (lvl + 1 == ss.trl_lvl) ? ss.trl_rank : ss.get_path(lvl + 1); + if (hit_count <= dsc) { + ss.follow = false; + descend = false; + } + } + if (descend) { + // Farther children first, so the last one dropped is the nearest. + for (uint32_t i = hit_count; i-- > dsc + 1; ) { + if (ss.entries.size() < VX_CFG_RTU_STACK_DEPTH) { + ss.entries.push_back({lvl + 1, i, hits[i].offset}); + } else { + ss.ovf = true; + ss.nd_lvl = lvl + 1; + ss.nd_rank = i; + } + } + current = hits[dsc].offset; + ++lvl; + ss.set_path(lvl, dsc); + if (ss.follow && lvl == ss.trl_lvl) { + ss.follow = false; } - current = hits[0].offset; - have_current = true; continue; } } - if (stack.empty()) { - have_current = false; - } else { - current = stack.back(); - stack.pop_back(); + if (ctx.terminated) return; + if (ss.ovf) { + ++perf.bvh_stack_restarts; + ss.restart = true; + return; } + if (ss.entries.size() == stack_floor) return; + ss.follow = false; + const ShortStack::Entry e = ss.entries.back(); + ss.entries.pop_back(); + current = e.off; + lvl = e.lvl; + ss.set_path(e.lvl, e.rank); } } @@ -711,8 +774,19 @@ WalkResult Bvh4Walker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, WalkCtx ctx = init_ctx(req, t, ro, rd, l); // Top-level (non-instanced) triangles carry no instance flags. - walk_bvh4_subtree(sv, ro, rd, root_off, 0, 0, 0, ctx, perf); - if (sv.miss) return {true, false}; + ShortStack ss; + ss.entries.reserve(VX_CFG_RTU_STACK_DEPTH); + for (;;) { + walk_bvh4_subtree(sv, ro, rd, root_off, 0, 0, 0, 0, ctx, ss, perf); + if (sv.miss) return {true, false}; + if (!ss.restart) break; + ss.entries.clear(); + ss.restart = false; + ss.ovf = false; + ss.follow = true; + ss.trl_lvl = ss.nd_lvl; + ss.trl_rank = ss.nd_rank; + } return {false, emit_lane_result(req, l, t, ctx)}; } diff --git a/sim/simx/tcu/tcu_tbuf.cpp b/sim/simx/tcu/tcu_tbuf.cpp index e80594297d..1f4834b0cf 100644 --- a/sim/simx/tcu/tcu_tbuf.cpp +++ b/sim/simx/tcu/tcu_tbuf.cpp @@ -71,7 +71,7 @@ struct Buffer { this->invalidate(); rsp_seen_ = false; next_tag_ = 0; - reads_ = 0; + // reads_ survives: it backs an MPM counter, which runs from device reset. } }; diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 5e2bd89173..46d16b471b 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -380,7 +380,7 @@ class TcuUnit::Impl { ~Impl() {} void reset() { - perf_stats_ = PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& sparse_meta : sparse_meta_) { std::fill(sparse_meta.begin(), sparse_meta.end(), 0); } diff --git a/sim/simx/tex/tex_core.cpp b/sim/simx/tex/tex_core.cpp index 960eeccca6..6f524fe548 100644 --- a/sim/simx/tex/tex_core.cpp +++ b/sim/simx/tex/tex_core.cpp @@ -96,7 +96,7 @@ class TexCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = TexCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& s : slots_) { s.in_use = false; s.pending_lines = 0; diff --git a/sw/kernel/scripts/link32.ld b/sw/kernel/scripts/link32.ld index b288bcff3b..433ed50de3 100644 --- a/sw/kernel/scripts/link32.ld +++ b/sw/kernel/scripts/link32.ld @@ -101,7 +101,7 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + .tdata : ALIGN(4) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) @@ -119,8 +119,10 @@ SECTIONS /* Per-hart stride for the TLS image, which spans .tdata + .tbss. vx_start.S places each hart's TLS block this far apart; using __tbss_size alone made adjacent harts' images overlap by - __tbss_offset bytes. __tbss_offset and __tbss_size are both - section-aligned, so their sum keeps the image alignment. */ + __tbss_offset bytes. .tdata is aligned like .tbss: when it is empty + the linker drops it and the TLS image starts at .tbss, so an unaligned + __tdata_start would skew __tbss_offset and the stride off the 4-byte + boundary that every hart's tp must keep. */ PROVIDE(__tls_block_size = __tbss_offset + __tbss_size); .preinit_array : { diff --git a/sw/kernel/scripts/link64.ld b/sw/kernel/scripts/link64.ld index aa6b87ff9c..2c92423941 100644 --- a/sw/kernel/scripts/link64.ld +++ b/sw/kernel/scripts/link64.ld @@ -97,7 +97,7 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + .tdata : ALIGN(8) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) @@ -115,8 +115,10 @@ SECTIONS /* Per-hart stride for the TLS image, which spans .tdata + .tbss. vx_start.S places each hart's TLS block this far apart; using __tbss_size alone made adjacent harts' images overlap by - __tbss_offset bytes. __tbss_offset and __tbss_size are both - section-aligned, so their sum keeps the image alignment. */ + __tbss_offset bytes. .tdata is aligned like .tbss: when it is empty + the linker drops it and the TLS image starts at .tbss, so an unaligned + __tdata_start would skew __tbss_offset and the stride off the 8-byte + boundary that every hart's tp must keep. */ PROVIDE(__tls_block_size = __tbss_offset + __tbss_size); .preinit_array : { diff --git a/sw/runtime/common/perf.cpp b/sw/runtime/common/perf.cpp index 4abbdc869e..8489a3a59d 100644 --- a/sw/runtime/common/perf.cpp +++ b/sw/runtime/common/perf.cpp @@ -161,6 +161,75 @@ struct CacheCounters { uint64_t mshr_st = 0; }; +struct RtuCounters { + uint64_t rays = 0; + uint64_t hits = 0; + uint64_t box_tests = 0; + uint64_t tri_tests = 0; + uint64_t callbacks = 0; + uint64_t busy = 0; + CacheCounters rtcache; + + RtuCounters& operator+=(const RtuCounters& rhs) { + rays += rhs.rays; + hits += rhs.hits; + box_tests += rhs.box_tests; + tri_tests += rhs.tri_tests; + callbacks += rhs.callbacks; + busy += rhs.busy; + rtcache.reads += rhs.rtcache.reads; + rtcache.miss_r += rhs.rtcache.miss_r; + return *this; + } +}; + +static vx_result_t query_rtu_counters(vx_device_h hdevice, uint32_t mpm_class, + uint32_t core_id, RtuCounters* c) { + const struct { + uint32_t addr; + uint64_t* value; + } fields[] = { + {VX_CSR_MPM_RTU_RAYS, &c->rays}, + {VX_CSR_MPM_RTU_HITS, &c->hits}, + {VX_CSR_MPM_RTU_BOX_TESTS, &c->box_tests}, + {VX_CSR_MPM_RTU_TRI_TESTS, &c->tri_tests}, + {VX_CSR_MPM_RTU_CALLBACKS, &c->callbacks}, + {VX_CSR_MPM_RTU_BUSY, &c->busy}, + {VX_CSR_MPM_RTCACHE_READS, &c->rtcache.reads}, + {VX_CSR_MPM_RTCACHE_MISS_R, &c->rtcache.miss_r}, + }; + for (const auto& f : fields) { + CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, f.addr, core_id, f.value), { return err; }); + } + return VX_SUCCESS; +} + +// Raw counters first (name=value), derived metrics in parentheses. `units` is +// the number of RTU sockets the counters were summed over, so the utilization +// stays a per-RTU average. +static void print_rtu_counters(FILE* stream, int core_id, const RtuCounters& c, + uint64_t cycles, uint64_t units) { + const double rays = (double)c.rays; + const double unit_cycles = (double)cycles * (double)units; + char buf[512]; + std::snprintf(buf, sizeof(buf), + "rtu: rays=%" PRIu64 ", hits=%" PRIu64 " (rate=%d%%), box_tests=%" PRIu64 " (%.1f/ray)" + ", tri_tests=%" PRIu64 " (%.1f/ray), callbacks=%" PRIu64 ", busy=%" PRIu64 " (util=%d%%)" + "; rtcache: reqs=%" PRIu64 " (%.1f/ray), miss=%" PRIu64 " (hit=%d%%)", + c.rays, c.hits, calc_percent(c.hits, c.rays), + c.box_tests, safe_div((double)c.box_tests, rays), + c.tri_tests, safe_div((double)c.tri_tests, rays), + c.callbacks, + c.busy, (int)std::lround(safe_div((double)c.busy, unit_cycles) * 100.0), + c.rtcache.reads, safe_div((double)c.rtcache.reads, rays), + c.rtcache.miss_r, calc_ratio(c.rtcache.miss_r, c.rtcache.reads)); + if (core_id >= 0) { + perf_print_core(stream, core_id, "%s", buf); + } else { + perf_print(stream, "%s", buf); + } +} + struct Metric { const char *label; int value; @@ -276,6 +345,7 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { const bool tex_en = (isa_flags & VX_ISA_EXT_TEX) != 0; const bool raster_en = (isa_flags & VX_ISA_EXT_RASTER) != 0; const bool om_en = (isa_flags & VX_ISA_EXT_OM) != 0; + const bool rtu_en = (isa_flags & VX_ISA_EXT_RTU) != 0; std::vector cores(num_cores); uint64_t total_instrs = 0; @@ -661,9 +731,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "TEX not supported"); break; } + // Socket-scoped counters: every core of a socket reads the same values. + const uint64_t num_sockets = (num_cores + socket_size - 1) / socket_size; uint64_t tot_reads = 0, tot_lat = 0, tot_st = 0; CacheCounters tcache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t s = 0; s < num_sockets; ++s) { + uint32_t core_id = s * (uint32_t)socket_size; + if (core_id >= num_cores) + break; uint64_t reads = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_TEX_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_TEX_LAT, core_id, &lat), { return err; }); @@ -696,9 +771,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "RASTER not supported"); break; } + // Cluster-scoped counters: every core of a cluster reads the same values. + const uint64_t cores_per_cluster = (num_cores + num_clusters - 1) / num_clusters; uint64_t tot_reads = 0, tot_lat = 0, tot_st = 0; CacheCounters rcache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t c = 0; c < num_clusters; ++c) { + uint32_t core_id = c * (uint32_t)cores_per_cluster; + if (core_id >= num_cores) + break; uint64_t reads = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_RASTER_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_RASTER_LAT, core_id, &lat), { return err; }); @@ -731,9 +811,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "OM not supported"); break; } + // Cluster-scoped counters: every core of a cluster reads the same values. + const uint64_t cores_per_cluster = (num_cores + num_clusters - 1) / num_clusters; uint64_t tot_reads = 0, tot_writes = 0, tot_lat = 0, tot_st = 0; CacheCounters ocache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t c = 0; c < num_clusters; ++c) { + uint32_t core_id = c * (uint32_t)cores_per_cluster; + if (core_id >= num_cores) + break; uint64_t reads = 0, writes = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_OM_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_OM_WRITES, core_id, &writes), { return err; }); @@ -770,6 +855,28 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { ocache_tot.mshr_st, calc_utility(ocache_tot.miss_r + ocache_tot.miss_w, ocache_tot.mshr_st)); } break; + case VX_DCR_MPM_CLASS_RTU: { + if (!rtu_en) { + perf_print(stream, "RTU not supported"); + break; + } + // Socket-scoped counters: every core of a socket reads the same values. + const uint64_t num_sockets = (num_cores + socket_size - 1) / socket_size; + RtuCounters tot; + for (uint32_t s = 0; s < num_sockets; ++s) { + uint32_t rep_core = s * (uint32_t)socket_size; + if (rep_core >= num_cores) + break; + RtuCounters c; + CHECK_ERR(query_rtu_counters(hdevice, mpm_class, rep_core, &c), { return err; }); + if (num_sockets > 1) { + print_rtu_counters(stream, rep_core, c, cores[rep_core].cycles, 1); + } + tot += c; + } + print_rtu_counters(stream, -1, tot, max_cycles, num_sockets); + } break; + default: fprintf(stream, "Error: invalid profiling class: %d)", mpm_class); return VX_ERR_INVALID_VALUE; diff --git a/tests/raytracing/rt_bvh_multinode/main.cpp b/tests/raytracing/rt_bvh_multinode/main.cpp index 9364557cd7..851ea7dbbb 100644 --- a/tests/raytracing/rt_bvh_multinode/main.cpp +++ b/tests/raytracing/rt_bvh_multinode/main.cpp @@ -62,6 +62,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp index cf0dd113c7..84662c9bb5 100644 --- a/tests/raytracing/rt_replay/main.cpp +++ b/tests/raytracing/rt_replay/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); device = nullptr; } diff --git a/tests/raytracing/rt_smoke/main.cpp b/tests/raytracing/rt_smoke/main.cpp index fc6bf196e3..6c020773b4 100644 --- a/tests/raytracing/rt_smoke/main.cpp +++ b/tests/raytracing/rt_smoke/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs/main.cpp b/tests/raytracing/rt_smoke_ahs/main.cpp index 5ce507f215..4fef4c6724 100644 --- a/tests/raytracing/rt_smoke_ahs/main.cpp +++ b/tests/raytracing/rt_smoke_ahs/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_bvh/main.cpp b/tests/raytracing/rt_smoke_ahs_bvh/main.cpp index b25a1db116..0e83e86bdf 100644 --- a/tests/raytracing/rt_smoke_ahs_bvh/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_bvh/main.cpp @@ -80,6 +80,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_custom/main.cpp b/tests/raytracing/rt_smoke_ahs_custom/main.cpp index 0f04741653..9f60a8f9ae 100644 --- a/tests/raytracing/rt_smoke_ahs_custom/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_custom/main.cpp @@ -68,6 +68,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_geom/main.cpp b/tests/raytracing/rt_smoke_ahs_geom/main.cpp index 01703d4331..c5417af0de 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/main.cpp @@ -69,6 +69,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_mixed/main.cpp b/tests/raytracing/rt_smoke_ahs_mixed/main.cpp index e0839e2322..106f892809 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_mixed/main.cpp @@ -71,6 +71,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_multi/main.cpp b/tests/raytracing/rt_smoke_ahs_multi/main.cpp index ab7176e2c4..b36ef03195 100644 --- a/tests/raytracing/rt_smoke_ahs_multi/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_multi/main.cpp @@ -81,6 +81,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_async_batch/main.cpp b/tests/raytracing/rt_smoke_async_batch/main.cpp index 4e4117bf90..3eaeea5346 100644 --- a/tests/raytracing/rt_smoke_async_batch/main.cpp +++ b/tests/raytracing/rt_smoke_async_batch/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh6/main.cpp b/tests/raytracing/rt_smoke_bvh6/main.cpp index 62d06a0788..9bf596ca66 100644 --- a/tests/raytracing/rt_smoke_bvh6/main.cpp +++ b/tests/raytracing/rt_smoke_bvh6/main.cpp @@ -65,6 +65,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_basic/main.cpp b/tests/raytracing/rt_smoke_bvh_basic/main.cpp index 4fbca70264..5cd9cce4bc 100644 --- a/tests/raytracing/rt_smoke_bvh_basic/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_basic/main.cpp @@ -65,6 +65,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp index 73ee543537..a0d51cdf0f 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp @@ -76,6 +76,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp b/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp index 5d5b835dce..d480019dee 100644 --- a/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp @@ -74,6 +74,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_chs/main.cpp b/tests/raytracing/rt_smoke_chs/main.cpp index 6e3b95c3ab..ad82c921ef 100644 --- a/tests/raytracing/rt_smoke_chs/main.cpp +++ b/tests/raytracing/rt_smoke_chs/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_cull_back/main.cpp b/tests/raytracing/rt_smoke_cull_back/main.cpp index 64f9b18677..83e03a524b 100644 --- a/tests/raytracing/rt_smoke_cull_back/main.cpp +++ b/tests/raytracing/rt_smoke_cull_back/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_cull_mask/main.cpp b/tests/raytracing/rt_smoke_cull_mask/main.cpp index 0664fdc290..3ac6722370 100644 --- a/tests/raytracing/rt_smoke_cull_mask/main.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_deep_stack/main.cpp b/tests/raytracing/rt_smoke_deep_stack/main.cpp index 77e900cbe9..205ffae302 100644 --- a/tests/raytracing/rt_smoke_deep_stack/main.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/main.cpp @@ -58,6 +58,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_deep_tlas/main.cpp b/tests/raytracing/rt_smoke_deep_tlas/main.cpp index e3eaf87fd4..a6281fe2e5 100644 --- a/tests/raytracing/rt_smoke_deep_tlas/main.cpp +++ b/tests/raytracing/rt_smoke_deep_tlas/main.cpp @@ -70,6 +70,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_fat_leaf/main.cpp b/tests/raytracing/rt_smoke_fat_leaf/main.cpp index 505cbb6a73..e13024778a 100644 --- a/tests/raytracing/rt_smoke_fat_leaf/main.cpp +++ b/tests/raytracing/rt_smoke_fat_leaf/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_host_cfg/main.cpp b/tests/raytracing/rt_smoke_host_cfg/main.cpp index 36f9e27a75..fa08bc677a 100644 --- a/tests/raytracing/rt_smoke_host_cfg/main.cpp +++ b/tests/raytracing/rt_smoke_host_cfg/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_inst_flags/main.cpp b/tests/raytracing/rt_smoke_inst_flags/main.cpp index 5d19e2b97c..40efd9cac5 100644 --- a/tests/raytracing/rt_smoke_inst_flags/main.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/main.cpp @@ -81,6 +81,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_is/main.cpp b/tests/raytracing/rt_smoke_is/main.cpp index f5b9f26c82..f82a0ae949 100644 --- a/tests/raytracing/rt_smoke_is/main.cpp +++ b/tests/raytracing/rt_smoke_is/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_miss/main.cpp b/tests/raytracing/rt_smoke_miss/main.cpp index a0a4774c6a..2d41da2c3f 100644 --- a/tests/raytracing/rt_smoke_miss/main.cpp +++ b/tests/raytracing/rt_smoke_miss/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_multitri/main.cpp b/tests/raytracing/rt_smoke_multitri/main.cpp index a3e8bc06ce..2efc8025c3 100644 --- a/tests/raytracing/rt_smoke_multitri/main.cpp +++ b/tests/raytracing/rt_smoke_multitri/main.cpp @@ -76,6 +76,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_partial_mask/main.cpp b/tests/raytracing/rt_smoke_partial_mask/main.cpp index e429da77dd..b65a00e2ba 100644 --- a/tests/raytracing/rt_smoke_partial_mask/main.cpp +++ b/tests/raytracing/rt_smoke_partial_mask/main.cpp @@ -51,6 +51,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_proc/main.cpp b/tests/raytracing/rt_smoke_proc/main.cpp index d5281bdd33..2e656172f8 100644 --- a/tests/raytracing/rt_smoke_proc/main.cpp +++ b/tests/raytracing/rt_smoke_proc/main.cpp @@ -61,6 +61,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_proc_inst/main.cpp b/tests/raytracing/rt_smoke_proc_inst/main.cpp index 539a8d3aa1..b8c0967c3a 100644 --- a/tests/raytracing/rt_smoke_proc_inst/main.cpp +++ b/tests/raytracing/rt_smoke_proc_inst/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_recursive/main.cpp b/tests/raytracing/rt_smoke_recursive/main.cpp index e9bfd26120..6be568cb6d 100644 --- a/tests/raytracing/rt_smoke_recursive/main.cpp +++ b/tests/raytracing/rt_smoke_recursive/main.cpp @@ -54,6 +54,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform/main.cpp b/tests/raytracing/rt_smoke_reform/main.cpp index fa2a191e63..29f32fabd0 100644 --- a/tests/raytracing/rt_smoke_reform/main.cpp +++ b/tests/raytracing/rt_smoke_reform/main.cpp @@ -75,6 +75,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform_mw/main.cpp b/tests/raytracing/rt_smoke_reform_mw/main.cpp index 27542a24a0..5d3984ce09 100644 --- a/tests/raytracing/rt_smoke_reform_mw/main.cpp +++ b/tests/raytracing/rt_smoke_reform_mw/main.cpp @@ -80,6 +80,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform_sbt/main.cpp b/tests/raytracing/rt_smoke_reform_sbt/main.cpp index 985f95fdbd..b3b4063b52 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/main.cpp +++ b/tests/raytracing/rt_smoke_reform_sbt/main.cpp @@ -82,6 +82,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_sbt/main.cpp b/tests/raytracing/rt_smoke_sbt/main.cpp index c87618f5da..d884f5c775 100644 --- a/tests/raytracing/rt_smoke_sbt/main.cpp +++ b/tests/raytracing/rt_smoke_sbt/main.cpp @@ -54,6 +54,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_shadow/main.cpp b/tests/raytracing/rt_smoke_shadow/main.cpp index 1f953ce076..f4e8926912 100644 --- a/tests/raytracing/rt_smoke_shadow/main.cpp +++ b/tests/raytracing/rt_smoke_shadow/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_tlas/main.cpp b/tests/raytracing/rt_smoke_tlas/main.cpp index aa7d38a850..f19f87b686 100644 --- a/tests/raytracing/rt_smoke_tlas/main.cpp +++ b/tests/raytracing/rt_smoke_tlas/main.cpp @@ -50,6 +50,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_tlas_builder/main.cpp b/tests/raytracing/rt_smoke_tlas_builder/main.cpp index 4e8e48bded..ff0f249bc8 100644 --- a/tests/raytracing/rt_smoke_tlas_builder/main.cpp +++ b/tests/raytracing/rt_smoke_tlas_builder/main.cpp @@ -57,6 +57,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } }