From aec93a41ff0817b3f93344d6a96173db431705bb Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 20:34:46 -0700 Subject: [PATCH 1/6] docs: fix the DXA MPM class and the perfetto converter path - dxa_async_copy_multicast.md listed VX_DCR_MPM_CLASS_DXA = 6; it is 16 in VX_types.toml. The perf CSR range is 0xB03-0xB08 (noslot_stalls was missing) and the VX_types.toml line reference is updated. - perfetto_analysis.md and the ci/perfetto.py usage text called the converter vortex_perfetto.py; it ships as ci/perfetto.py. Co-Authored-By: Claude Opus 5.5 --- ci/perfetto.py | 8 ++++---- docs/designs/dxa_async_copy_multicast.md | 8 ++++---- docs/perfetto_analysis.md | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/perfetto.py b/ci/perfetto.py index 9b554f47fa..cf1720dbd9 100755 --- a/ci/perfetto.py +++ b/ci/perfetto.py @@ -14,7 +14,7 @@ # limitations under the License. """ -vortex_perfetto.py +perfetto.py Convert Vortex GPU RTL or SimX log traces into a Perfetto-compatible trace (Chrome Trace JSON). @@ -30,13 +30,13 @@ Usage examples # SimX (default), compressed: - python3 vortex_perfetto.py run_simx.log -c -o vortex.perfetto.json.gz + python3 ci/perfetto.py run_simx.log -c -o vortex.perfetto.json.gz # RTL sim: - python3 vortex_perfetto.py run.log -t rtlsim -c -o vortex.perfetto.json.gz + python3 ci/perfetto.py run.log -t rtlsim -c -o vortex.perfetto.json.gz # Limit export to a cycle window: - python3 vortex_perfetto.py run.log -t rtlsim --cycle-min 10000 --cycle-max 20000 -c -o win.json.gz + python3 ci/perfetto.py run.log -t rtlsim --cycle-min 10000 --cycle-max 20000 -c -o win.json.gz Open in Perfetto UI - Web UI: https://ui.perfetto.dev (load the .json or .json.gz) diff --git a/docs/designs/dxa_async_copy_multicast.md b/docs/designs/dxa_async_copy_multicast.md index 470a813472..fcb0c831eb 100644 --- a/docs/designs/dxa_async_copy_multicast.md +++ b/docs/designs/dxa_async_copy_multicast.md @@ -58,10 +58,10 @@ There is no software→global path; DXA is GMEM-read / LMEM-write only. `VX_CFG_NUM_DXA_UNITS = max(1, ceil(NUM_CORES/8))`, `VX_CFG_DXA_QUEUE_SIZE = 16`, `VX_CFG_DXA_MAX_INFLIGHT = 8`, `VX_DCR_DXA_DESC_COUNT = 16`. -- **Perf CSRs** `[csr_mpm_dxa]` 0xB03–0xB07 (transfers, gmem_reads, - gmem_dedup, lmem_writes, gmem_latency), - `VX_DCR_MPM_CLASS_DXA = 6` - ([`VX_types.toml:557-568`](../../VX_types.toml#L557)). +- **Perf CSRs** `[csr_mpm_dxa]` 0xB03–0xB08 (transfers, gmem_reads, + gmem_dedup, lmem_writes, gmem_latency, noslot_stalls), + `VX_DCR_MPM_CLASS_DXA = 16` + ([`VX_types.toml:767-780`](../../VX_types.toml#L767)). - **Cluster dispatch** (multicast prerequisite): `VX_CSR_CTA_CLUSTER_SIZE = 0xCE0`, `VX_DCR_KMU_CLUSTER_DIM_{X,Y,Z}` diff --git a/docs/perfetto_analysis.md b/docs/perfetto_analysis.md index 843b8b4dec..e0217b81b6 100644 --- a/docs/perfetto_analysis.md +++ b/docs/perfetto_analysis.md @@ -1,6 +1,6 @@ # Vortex Perfetto Trace: Performance Analysis Guide -This guide explains how to **generate** and **interpret** a Vortex GPU trace that has been converted into **Perfetto-compatible Chrome Trace JSON** using `vortex_perfetto.py`. +This guide explains how to **generate** and **interpret** a Vortex GPU trace that has been converted into **Perfetto-compatible Chrome Trace JSON** using `ci/perfetto.py`. It is written for people who are: - New to Perfetto @@ -33,13 +33,13 @@ This guide focuses on **Vortex GPU 1**. #### SimX (default) ```bash -python3 vortex_perfetto.py run_simx.log -t simx -c -o vortex.perfetto.json.gz +python3 ci/perfetto.py run_simx.log -t simx -c -o vortex.perfetto.json.gz ``` #### RTL simulation ```bash -python3 vortex_perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.gz +python3 ci/perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.gz ``` ### Common options @@ -47,7 +47,7 @@ python3 vortex_perfetto.py run_rtlsim.log -t rtlsim -c -o vortex.perfetto.json.g - **Limit export to a cycle/tick window** (highly recommended for huge logs): ```bash -python3 vortex_perfetto.py run_rtlsim.log -t rtlsim \ +python3 ci/perfetto.py run_rtlsim.log -t rtlsim \ --cycle-min 10000 --cycle-max 20000 -c -o window.json.gz ``` From 26a19715ce50df55309b1f8eb7eb5a2de0faaadd Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 20:34:46 -0700 Subject: [PATCH 2/6] perf: read TEX counters per socket, RASTER/OM per cluster The texture unit and its tcache are socket resources (VX_socket); the rasterizer, OM unit and their caches are cluster resources (VX_cluster). Every core of a socket/cluster reads the same MPM values. - SimX read the TEX counters from the cluster, so with 2 sockets each core reported the sum of both sockets. - vx_device_dump_perf summed the TEX, RASTER and OM counters over every core, so the totals were multiplied by SOCKET_SIZE (TEX) or by the cores per cluster (RASTER, OM). It now reads one core per socket or cluster, as the icache/dcache and L2/DXA printers already do. gfx_tex, 2 cores, SOCKET_SIZE=2 (4096 texel reads): reported 8192, now 4096. Cycle counts unchanged. Co-Authored-By: Claude Opus 5.5 --- sim/simx/csr_unit.cpp | 16 ++++++++-------- sw/runtime/common/perf.cpp | 21 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index c000ef2fa5..550ec14fca 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -263,15 +263,15 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { #endif #ifdef VX_CFG_EXT_TEX_ENABLE case VX_DCR_MPM_CLASS_TEX: { - auto cluster_perf = core_->socket()->cluster()->perf_stats(); + auto socket_perf = core_->socket()->perf_stats(); switch (addr) { - CSR_READ_64(VX_CSR_MPM_TEX_READS, cluster_perf.tex.mem_reads); - CSR_READ_64(VX_CSR_MPM_TEX_LAT, cluster_perf.tex.mem_latency); - CSR_READ_64(VX_CSR_MPM_TEX_ST, cluster_perf.tex.stall_cycles); - CSR_READ_64(VX_CSR_MPM_TCACHE_READS, cluster_perf.tcache.reads); - CSR_READ_64(VX_CSR_MPM_TCACHE_MISS_R, cluster_perf.tcache.read_misses); - CSR_READ_64(VX_CSR_MPM_TCACHE_BANK_ST,cluster_perf.tcache.bank_stalls); - CSR_READ_64(VX_CSR_MPM_TCACHE_MSHR_ST,cluster_perf.tcache.mshr_stalls); + CSR_READ_64(VX_CSR_MPM_TEX_READS, socket_perf.tex.mem_reads); + CSR_READ_64(VX_CSR_MPM_TEX_LAT, socket_perf.tex.mem_latency); + CSR_READ_64(VX_CSR_MPM_TEX_ST, socket_perf.tex.stall_cycles); + CSR_READ_64(VX_CSR_MPM_TCACHE_READS, socket_perf.tcache.reads); + CSR_READ_64(VX_CSR_MPM_TCACHE_MISS_R, socket_perf.tcache.read_misses); + CSR_READ_64(VX_CSR_MPM_TCACHE_BANK_ST,socket_perf.tcache.bank_stalls); + CSR_READ_64(VX_CSR_MPM_TCACHE_MSHR_ST,socket_perf.tcache.mshr_stalls); } } break; #endif diff --git a/sw/runtime/common/perf.cpp b/sw/runtime/common/perf.cpp index 4abbdc869e..cd357ab8fa 100644 --- a/sw/runtime/common/perf.cpp +++ b/sw/runtime/common/perf.cpp @@ -661,9 +661,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "TEX not supported"); break; } + // Socket-scoped counters: every core of a socket reads the same values. + const uint64_t num_sockets = (num_cores + socket_size - 1) / socket_size; uint64_t tot_reads = 0, tot_lat = 0, tot_st = 0; CacheCounters tcache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t s = 0; s < num_sockets; ++s) { + uint32_t core_id = s * (uint32_t)socket_size; + if (core_id >= num_cores) + break; uint64_t reads = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_TEX_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_TEX_LAT, core_id, &lat), { return err; }); @@ -696,9 +701,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "RASTER not supported"); break; } + // Cluster-scoped counters: every core of a cluster reads the same values. + const uint64_t cores_per_cluster = (num_cores + num_clusters - 1) / num_clusters; uint64_t tot_reads = 0, tot_lat = 0, tot_st = 0; CacheCounters rcache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t c = 0; c < num_clusters; ++c) { + uint32_t core_id = c * (uint32_t)cores_per_cluster; + if (core_id >= num_cores) + break; uint64_t reads = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_RASTER_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_RASTER_LAT, core_id, &lat), { return err; }); @@ -731,9 +741,14 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { perf_print(stream, "OM not supported"); break; } + // Cluster-scoped counters: every core of a cluster reads the same values. + const uint64_t cores_per_cluster = (num_cores + num_clusters - 1) / num_clusters; uint64_t tot_reads = 0, tot_writes = 0, tot_lat = 0, tot_st = 0; CacheCounters ocache_tot; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + for (uint32_t c = 0; c < num_clusters; ++c) { + uint32_t core_id = c * (uint32_t)cores_per_cluster; + if (core_id >= num_cores) + break; uint64_t reads = 0, writes = 0, lat = 0, st = 0; CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_OM_READS, core_id, &reads), { return err; }); CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_OM_WRITES, core_id, &writes), { return err; }); From c16b8fdfe0e6b2de768fd5751379cb3c50cf7c12 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 20:34:46 -0700 Subject: [PATCH 3/6] simx: keep MPM counters across kernel launches The MPM counters are free-running from device reset: rtlsim only pulses start between launches, and the SimX core already keeps its counters for that reason. The caches, local memory, TEX, RASTER, OM, TCU (and its tile buffer), DXA and the memory counters still cleared theirs in the per-launch reset(), so a dump after several launches reported only the last one. vecadd launched twice, --perf=4: dcache reqs SimX 224 -> 448 (rtlsim 450), memory reqs 20 -> 40. Instruction and cycle counts are unchanged. Co-Authored-By: Claude Opus 5.5 --- sim/simx/dxa/dxa_core.cpp | 2 +- sim/simx/mem/cache.cpp | 2 +- sim/simx/mem/local_mem.cpp | 2 +- sim/simx/om/om_core.cpp | 2 +- sim/simx/processor.cpp | 4 +--- sim/simx/processor_impl.h | 8 ++++---- sim/simx/raster/raster_core.cpp | 2 +- sim/simx/tcu/tcu_tbuf.cpp | 2 +- sim/simx/tcu/tcu_unit.cpp | 2 +- sim/simx/tex/tex_core.cpp | 2 +- 10 files changed, 13 insertions(+), 15 deletions(-) diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 5f3c357644..1af9074776 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -138,7 +138,7 @@ class DxaCore::Impl { void reset() { cycle_ = 0; queue_.clear(); - perf_stats_ = DxaCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& w : workers_) { w.state = WState::IDLE; w.work_list.clear(); diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 8c4dcfc0a8..df1a19d7ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -688,7 +688,7 @@ class CacheBank : public SimObject { protected: void on_reset() { - perf_stats_ = Cache::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. pending_mshr_size_ = 0; pending_amo_probes_ = 0; pipe_fill_count_ = 0; diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index d2b6d60598..41b82e8575 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -99,7 +99,7 @@ class LocalMem::Impl { virtual ~Impl() {} void reset() { - perf_stats_ = PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); std::fill(amo_busy_.begin(), amo_busy_.end(), 0); diff --git a/sim/simx/om/om_core.cpp b/sim/simx/om/om_core.cpp index ef8a911e8f..d7bf17aad6 100644 --- a/sim/simx/om/om_core.cpp +++ b/sim/simx/om/om_core.cpp @@ -128,7 +128,7 @@ class OmCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = OmCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& s : slots_) { s.in_use = false; s.state = State::ADDR; diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 52ad76ca1a..cc9116cfb3 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -331,9 +331,7 @@ void ProcessorImpl::forward_delegated_launch() { void ProcessorImpl::reset() { SimPlatform::instance().reset(); - perf_mem_reads_ = 0; - perf_mem_writes_ = 0; - perf_mem_latency_ = 0; + // The perf_mem_* counters survive: the MPM counters run from device reset. perf_mem_pending_reads_ = 0; is_cycle_initialized_ = false; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index c9c7c7692d..ebe59cf297 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -95,10 +95,10 @@ class ProcessorImpl { #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_; - uint64_t perf_mem_reads_; - uint64_t perf_mem_writes_; - uint64_t perf_mem_latency_; - uint64_t perf_mem_pending_reads_; + uint64_t perf_mem_reads_ = 0; + uint64_t perf_mem_writes_ = 0; + uint64_t perf_mem_latency_ = 0; + uint64_t perf_mem_pending_reads_ = 0; // Tracks whether cycle() has done its first-call init (reset + // kmu_->start()). reset() clears it so a back-to-back kernel launch // via cycle() re-dispatches the KMU. diff --git a/sim/simx/raster/raster_core.cpp b/sim/simx/raster/raster_core.cpp index 9b7b16cc22..4cb79ad943 100644 --- a/sim/simx/raster/raster_core.cpp +++ b/sim/simx/raster/raster_core.cpp @@ -152,7 +152,7 @@ class RasterCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = RasterCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. reset_load_state(); state_ = State::IDLE; // The per-launch reset precedes the delegated launch's frame kick, so both diff --git a/sim/simx/tcu/tcu_tbuf.cpp b/sim/simx/tcu/tcu_tbuf.cpp index e80594297d..1f4834b0cf 100644 --- a/sim/simx/tcu/tcu_tbuf.cpp +++ b/sim/simx/tcu/tcu_tbuf.cpp @@ -71,7 +71,7 @@ struct Buffer { this->invalidate(); rsp_seen_ = false; next_tag_ = 0; - reads_ = 0; + // reads_ survives: it backs an MPM counter, which runs from device reset. } }; diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 5e2bd89173..46d16b471b 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -380,7 +380,7 @@ class TcuUnit::Impl { ~Impl() {} void reset() { - perf_stats_ = PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& sparse_meta : sparse_meta_) { std::fill(sparse_meta.begin(), sparse_meta.end(), 0); } diff --git a/sim/simx/tex/tex_core.cpp b/sim/simx/tex/tex_core.cpp index 960eeccca6..6f524fe548 100644 --- a/sim/simx/tex/tex_core.cpp +++ b/sim/simx/tex/tex_core.cpp @@ -96,7 +96,7 @@ class TexCore::Impl { void reset() { cycle_ = 0; - perf_stats_ = TexCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset. for (auto& s : slots_) { s.in_use = false; s.pending_lines = 0; From 0c6c9876e01d8174186c797b22bc97ba4412e6fe Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 21:10:03 -0700 Subject: [PATCH 4/6] rtu: performance counters (MPM class 15) Adds the ray-tracing unit to the MPM counters, in SimX and RTL, read with --perf=15. The set is limited to quantities every BVH ray-tracing unit has, whatever its internal organization, so the counters stay meaningful as the RTU evolves; the class's CSR window keeps room for more: rays rays traced (active lanes of the trace instructions) hits rays whose final result is a hit (misses = rays - hits) box_tests ray/box intersection tests tri_tests ray/triangle intersection tests callbacks any-hit and intersection candidates returned to the shader busy cycles with at least one ray in flight rtcache reads and read misses (the RTU's memory demand) CSRs 0xB03-0xB0A. The counters are socket scoped: the RTU and its rtcache live in the socket, which sums its RTU cores. - RTL: box/triangle test strobes from the scheduler, ray, hit and callback lane counts and busy in the core, summed in VX_socket and read through VX_csr_data. All of it is under PERF_ENABLE. The DBG_RTU_OCC debug block is removed. - SimX: the same events (rays_issued, which counted traces, is renamed). The walker now models the hardware's bounded short stack with trail restarts instead of a flat restart charge, so box and triangle tests include the re-descents of restarted walks as in the RTL; this changes timing only on restart scenes (rt_smoke_deep_stack 8091 -> 9098 cycles). rt_raycast, the only raytracing perf baseline, is unchanged. The counters survive the per-launch reset, like the other MPM counters. - Runtime: vx_device_dump_perf prints the class per socket and in total, with hit rate, tests per ray, memory requests per ray, utilization and rtcache hit rate; every raytracing test now dumps its counters. - CI: model_parity cases can compare an MPM class's counters (parity_counters); model_parity-rt_raycast checks rays, hits, box_tests, tri_tests and callbacks exactly, and the counter invariants (hits <= rays, busy <= cycles). - Docs: counter definitions in ray_tracing_architecture.md. Validation: model_parity-rt_raycast passes (instructions equal, cycles within 1.92%, the five work counters exact) and failed as expected with the box-test strobe removed; callbacks match on rt_smoke_reform_sbt (4 on both models); tests/regression 88/88 on SimX and rtlsim with instruction and cycle counts equal to master; Verilator -Wall clean at XLEN 32 and 64. Co-Authored-By: Claude Opus 5.5 --- VX_types.toml | 20 ++ ci/test_runner.py | 60 +++++- ci/testcase.py | 17 ++ ci/testcases/raytracing.yaml | 6 + docs/designs/graphics_hardware_stack.md | 3 +- docs/designs/ray_tracing_architecture.md | 47 ++++- hw/rtl/VX_gpu_pkg.sv | 15 ++ hw/rtl/VX_socket.sv | 33 +++- hw/rtl/core/VX_csr_data.sv | 15 ++ hw/rtl/rtu/VX_rtu_core.sv | 99 ++++++---- hw/rtl/rtu/VX_rtu_pkg.sv | 7 + hw/rtl/rtu/VX_rtu_scheduler.sv | 9 + hw/unittest/rtu_core/VX_rtu_core_top.sv | 3 + sim/simx/csr_unit.cpp | 15 ++ sim/simx/rtu/rtu_core.cpp | 32 ++-- sim/simx/rtu/rtu_types.h | 16 +- sim/simx/rtu/rtu_walker.cpp | 178 +++++++++++++----- sw/runtime/common/perf.cpp | 92 +++++++++ tests/raytracing/rt_bvh_multinode/main.cpp | 1 + tests/raytracing/rt_replay/main.cpp | 1 + tests/raytracing/rt_smoke/main.cpp | 1 + tests/raytracing/rt_smoke_ahs/main.cpp | 1 + tests/raytracing/rt_smoke_ahs_bvh/main.cpp | 1 + tests/raytracing/rt_smoke_ahs_custom/main.cpp | 1 + tests/raytracing/rt_smoke_ahs_geom/main.cpp | 1 + tests/raytracing/rt_smoke_ahs_mixed/main.cpp | 1 + tests/raytracing/rt_smoke_ahs_multi/main.cpp | 1 + .../raytracing/rt_smoke_async_batch/main.cpp | 1 + tests/raytracing/rt_smoke_bvh6/main.cpp | 1 + tests/raytracing/rt_smoke_bvh_basic/main.cpp | 1 + .../rt_smoke_bvh_instanced/main.cpp | 1 + .../rt_smoke_bvh_multilevel/main.cpp | 1 + tests/raytracing/rt_smoke_chs/main.cpp | 1 + tests/raytracing/rt_smoke_cull_back/main.cpp | 1 + tests/raytracing/rt_smoke_cull_mask/main.cpp | 1 + tests/raytracing/rt_smoke_deep_stack/main.cpp | 1 + tests/raytracing/rt_smoke_deep_tlas/main.cpp | 1 + tests/raytracing/rt_smoke_fat_leaf/main.cpp | 1 + tests/raytracing/rt_smoke_host_cfg/main.cpp | 1 + tests/raytracing/rt_smoke_inst_flags/main.cpp | 1 + tests/raytracing/rt_smoke_is/main.cpp | 1 + tests/raytracing/rt_smoke_miss/main.cpp | 1 + tests/raytracing/rt_smoke_multitri/main.cpp | 1 + .../raytracing/rt_smoke_partial_mask/main.cpp | 1 + tests/raytracing/rt_smoke_proc/main.cpp | 1 + tests/raytracing/rt_smoke_proc_inst/main.cpp | 1 + tests/raytracing/rt_smoke_recursive/main.cpp | 1 + tests/raytracing/rt_smoke_reform/main.cpp | 1 + tests/raytracing/rt_smoke_reform_mw/main.cpp | 1 + tests/raytracing/rt_smoke_reform_sbt/main.cpp | 1 + tests/raytracing/rt_smoke_sbt/main.cpp | 1 + tests/raytracing/rt_smoke_shadow/main.cpp | 1 + tests/raytracing/rt_smoke_tlas/main.cpp | 1 + .../raytracing/rt_smoke_tlas_builder/main.cpp | 1 + 54 files changed, 586 insertions(+), 117 deletions(-) diff --git a/VX_types.toml b/VX_types.toml index ff3437215e..80ec9c17f6 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -848,6 +848,26 @@ VX_CSR_MPM_OCACHE_BANK_ST_H = 0xB8B VX_CSR_MPM_OCACHE_MSHR_ST = 0xB0C # MSHR stalls VX_CSR_MPM_OCACHE_MSHR_ST_H = 0xB8C +[csr_mpm_rtu] +# PERF: ray-tracing unit (per-socket; summed across the socket's RTU cores) +VX_CSR_MPM_RTU_RAYS = 0xB03 # rays traced (active lanes of the trace instructions) +VX_CSR_MPM_RTU_RAYS_H = 0xB83 +VX_CSR_MPM_RTU_HITS = 0xB04 # rays whose final result is a hit +VX_CSR_MPM_RTU_HITS_H = 0xB84 +VX_CSR_MPM_RTU_BOX_TESTS = 0xB05 # ray/box intersection tests +VX_CSR_MPM_RTU_BOX_TESTS_H = 0xB85 +VX_CSR_MPM_RTU_TRI_TESTS = 0xB06 # ray/triangle intersection tests +VX_CSR_MPM_RTU_TRI_TESTS_H = 0xB86 +VX_CSR_MPM_RTU_CALLBACKS = 0xB07 # any-hit and intersection candidates returned to the shader +VX_CSR_MPM_RTU_CALLBACKS_H = 0xB87 +VX_CSR_MPM_RTU_BUSY = 0xB08 # cycles with a ray in flight +VX_CSR_MPM_RTU_BUSY_H = 0xB88 +# PERF: ray-tracing cache +VX_CSR_MPM_RTCACHE_READS = 0xB09 # total reads +VX_CSR_MPM_RTCACHE_READS_H = 0xB89 +VX_CSR_MPM_RTCACHE_MISS_R = 0xB0A # read misses +VX_CSR_MPM_RTCACHE_MISS_R_H = 0xB8A + # XLEN is an environment-supplied build axis (configure exports it); the # [memmap]/[vm] exprs above reference it. Declared as a [[builtin]] so the # dependency is explicit and typed; defaults to 0 (→ 32-bit) when unset. diff --git a/ci/test_runner.py b/ci/test_runner.py index 2cf559d5f1..5feae9f5d6 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -16,6 +16,7 @@ instructions must match exactly (both are deterministic ISA-level executions, so any delta is functional divergence) and cycles must agree within the case's `tolerance` (SimX is the timing model of the RTL, not just a functional oracle). +A case with `parity_counters` also compares that MPM class's counters. """ import os @@ -32,7 +33,7 @@ def _run_one(case, xlen, driver): - """One parity leg: run under `driver`, return (instrs, cycles).""" + """One parity leg: run under `driver`, return (instrs, cycles, output).""" argv, env = case.run_command(xlen, driver=driver) rc, out = tc.execute_capture(argv, env) assert rc == 0, "{} [{}] failed (exit {}): {}".format( @@ -41,12 +42,59 @@ def _run_one(case, xlen, driver): assert perf, "{} [{}]: no 'PERF: instrs=…, cycles=…' summary in output " \ "(app must call vx_device_dump_perf)".format(case.id, driver) instrs, cycles = perf[-1] - return int(instrs), int(cycles) + return int(instrs), int(cycles), out + + +def parse_counters(out, prefix): + """The raw `name=value` counters of an MPM class's device-total line + (`PERF: : ...`; per-unit lines carry a `coreN:` prefix and do not + match). Derived metrics in parentheses are dropped.""" + lines = re.findall(r"^PERF: {}: (.*)$".format(re.escape(prefix)), out, re.M) + if not lines: + return None + raw = re.sub(r"\([^)]*\)", "", lines[-1].split(";")[0]) + return {k: int(v) for k, v in re.findall(r"(\w+)=(\d+)", raw)} + + +def rtu_invariants(c, cycles): + """Violations of the RTU counters' definitions (empty when consistent).""" + errs = [] + if c["hits"] > c["rays"]: + errs.append("hits {} > rays {}".format(c["hits"], c["rays"])) + if c["busy"] > cycles: + errs.append("busy {} > cycles {}".format(c["busy"], cycles)) + return errs + + +def _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles): + pc = case.parity_counters + prefix = pc["prefix"] + sc = parse_counters(simx_out, prefix) + rc = parse_counters(rtl_out, prefix) + assert sc and rc, "{}: no 'PERF: {}: ...' counter line (simx={}, rtlsim={})".format( + case.id, prefix, sc is not None, rc is not None) + if prefix == "rtu": + for drv, c, cyc in (("simx", sc, simx_cycles), ("rtlsim", rc, rtl_cycles)): + errs = rtu_invariants(c, cyc) + assert not errs, "{} [{}]: RTU counter invariants violated: {}".format( + case.id, drv, "; ".join(errs)) + fails = [] + for name in pc.get("exact", []): + print("PARITY: {}: {} simx={} rtlsim={}".format(case.id, name, sc.get(name), rc.get(name))) + if name not in sc or sc[name] != rc.get(name): + fails.append("{} simx={} rtlsim={}".format(name, sc.get(name), rc.get(name))) + for name in pc.get("tolerance", []): + s, r = sc.get(name), rc.get(name) + gap = abs(r - s) / float(max(r, 1)) if (s is not None and r is not None) else 1.0 + print("PARITY: {}: {} simx={} rtlsim={} gap={:.2%}".format(case.id, name, s, r, gap)) + if gap > case.tolerance: + fails.append("{} simx={} rtlsim={} gap={:.2%}".format(name, s, r, gap)) + assert not fails, "{}: {} counter mismatch: {}".format(case.id, prefix, "; ".join(fails)) def _model_parity(case, xlen): - simx_instrs, simx_cycles = _run_one(case, xlen, "simx") - rtl_instrs, rtl_cycles = _run_one(case, xlen, "rtlsim") + simx_instrs, simx_cycles, simx_out = _run_one(case, xlen, "simx") + rtl_instrs, rtl_cycles, rtl_out = _run_one(case, xlen, "rtlsim") gap = abs(rtl_cycles - simx_cycles) / float(rtl_cycles) print("PARITY: {}: instrs simx={} rtlsim={}, cycles simx={} rtlsim={}, " "gap={:.2%} (tolerance {:.0%})".format( @@ -58,11 +106,13 @@ def _model_parity(case, xlen): assert gap <= case.tolerance, \ "{}: cycle gap {:.2%} exceeds tolerance {:.0%} (simx={}, rtlsim={})".format( case.id, gap, case.tolerance, simx_cycles, rtl_cycles) + if case.parity_counters: + _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles) def _perf_gate(case, xlen, update): """Run the benchmark on rtlsim; record or compare its cycles vs baseline.""" - instrs, cycles = _run_one(case, xlen, "rtlsim") + instrs, cycles, _ = _run_one(case, xlen, "rtlsim") if update: pb.record(case, xlen, cycles, instrs) print("PERF-BASELINE: {} xlen={} record cycles={} instrs={}".format( diff --git a/ci/testcase.py b/ci/testcase.py index 0e8da16c65..c9845ca45a 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -108,6 +108,11 @@ def __init__(self, category, entry, driver, defaults): self.xlens = [32] self.tolerance = float(entry.get("tolerance", defaults.get("tolerance", DEFAULT_PARITY_TOLERANCE))) + # model_parity can also compare an MPM class's counters, read from the + # runtime's `PERF: : name=value, ...` device-total line (the run + # must pass the class with --perf=N): `exact` counters must match, and + # `tolerance` counters must agree within the case tolerance. + self.parity_counters = dict(entry.get("parity_counters", {})) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) # make-run / script fields self.dir = entry.get("dir", "") @@ -585,6 +590,18 @@ def cmd_lint(args): "(the runner drives simx and rtlsim)".format(c.id)) if not (0.0 < c.tolerance < 1.0): errors.append("{}: tolerance must be in (0, 1)".format(c.id)) + if c.parity_counters: + pc = c.parity_counters + if c.check != "model_parity": + errors.append("{}: parity_counters needs check: model_parity".format(c.id)) + if not isinstance(pc.get("prefix"), str) or not pc.get("prefix"): + errors.append("{}: parity_counters needs a 'prefix'".format(c.id)) + if set(pc) - {"prefix", "exact", "tolerance"}: + errors.append("{}: parity_counters keys are prefix/exact/tolerance".format(c.id)) + if not (pc.get("exact") or pc.get("tolerance")): + errors.append("{}: parity_counters compares nothing".format(c.id)) + if "--perf=" not in c.flags: + errors.append("{}: parity_counters needs --perf= in flags".format(c.id)) if errors: for e in errors: sys.stderr.write("LINT ERROR: " + e + "\n") diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index 734ef894cf..66c1e171db 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -85,6 +85,12 @@ tests: via: blackbox app: rt_raycast configs: -DVX_CFG_EXT_RTU_ENABLE + # The RTU counters too: the traversal work both models must do identically. + # busy is a timing counter and is left out. + flags: --perf=15 + parity_counters: + prefix: rtu + exact: [rays, hits, box_tests, tri_tests, callbacks] # Perf-regression twin of the model_parity-rt_raycast benchmark (same run, cycles-vs-baseline). - id: perf_gate-rt_raycast diff --git a/docs/designs/graphics_hardware_stack.md b/docs/designs/graphics_hardware_stack.md index 2b90dcd737..43132045a6 100644 --- a/docs/designs/graphics_hardware_stack.md +++ b/docs/designs/graphics_hardware_stack.md @@ -104,7 +104,8 @@ been **retired** in favor of this push/launch path (§4). rides the aperture address. DCRs are broadcast to all cluster instances; each raster instance self-selects its tile stripe by `INSTANCE_IDX`. -- **Perf** MPM classes RASTER=12, TEX=13, OM=14. +- **Perf** MPM classes RASTER=12, TEX=13, OM=14, RTU=15 (ray tracing, see + [ray_tracing_architecture.md §8.1](ray_tracing_architecture.md#81-performance-counters-mpm-class-15)). ### 2.1 The graphics register window (FF↔SIMT handoff) diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index 07b57b3318..24a7a3496d 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -467,10 +467,49 @@ oracle**: SimX additionally models **per-warp multi-trace concurrency** (several traces in flight from one warp), which the RTL core does not implement — SimX remains the -oracle for that. Perf: MPM class `RTU` plus the SimX `PerfStats` (context/slot -occupancy, front-end busy ticks, box/tri test counts); the RTL core keeps its own -per-slot occupancy counters (busy / write / callback / fill and the two idle -causes: idle-with-a-ray vs starved) under `DBG_RTU_OCC`. +oracle for that. Its exploration counters (context occupancy, fetch merging, +walker cycle accounting) stay in `PerfStats` and the `VX_RTU_STATS=1` teardown +dump; the architectural set below is what both models expose. + +### 8.1 Performance counters (MPM class 15) + +`--perf=15` (`VORTEX_PROFILING=15`, or `vx_device_mpm_query` with +`VX_DCR_MPM_CLASS_RTU`) reads the RTU counters on every driver. They are +**socket-scoped**: a socket sums its RTU cores and hands the result to each of its +cores' CSR file, so every core of a socket reads the same values. The runtime +queries one core per socket, prints a per-socket line when there is more than one +socket, and always prints the device total. + +The set is kept to quantities every BVH ray-tracing unit has, whatever its +internal organization, so the counters stay meaningful as the RTU evolves. +Microarchitecture-specific measures (slots, short-stack restarts, front-end steps) +are left to `PerfStats`; the class's CSR window (B03–B1F) leaves room for +future counters. + +| CSR | Counter | Definition | +|---|---|---| +| B03 | `RTU_RAYS` | rays traced: the active lanes of the trace instructions | +| B04 | `RTU_HITS` | rays whose final result is a hit (misses = rays − hits) | +| B05 | `RTU_BOX_TESTS` | ray/box intersection tests (node children and procedural boxes) | +| B06 | `RTU_TRI_TESTS` | ray/triangle intersection tests | +| B07 | `RTU_CALLBACKS` | any-hit and intersection candidates returned to the shader (one per ray per candidate) | +| B08 | `RTU_BUSY` | cycles with at least one ray in flight | +| B09 | `RTCACHE_READS` | RTU memory requests (RTCache reads) | +| B0A | `RTCACHE_MISS_R` | RTCache read misses | + +Box and triangle tests include the re-descents of restarted walks: SimX runs the +same `VX_CFG_RTU_STACK_DEPTH` short stack and trail restart as the RTL, so both +models do the same intersection work. Invariants: `RTU_HITS ≤ RTU_RAYS`, +`RTU_BUSY ≤ cycles`. + +The `model_parity-rt_raycast` case requires rays, hits, box tests, triangle tests +and callbacks to match SimX exactly. `RTU_BUSY` and the RTCache counters are +timing and memory-system measures and are not gated: SimX keeps a ray's fetched +lines where the RTL refetches per structure access. + +The runtime also derives the hit rate, box and triangle tests per ray, memory +requests per ray, RTU utilization and the RTCache hit rate. In RTL the counters +exist only in `PERF_ENABLE` builds. --- diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 53193421c1..88a1b908a4 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1316,6 +1316,17 @@ package VX_gpu_pkg; } om_perf_t; `endif +`ifdef VX_CFG_EXT_RTU_ENABLE + typedef struct packed { + logic [PERF_CTR_BITS-1:0] rays; + logic [PERF_CTR_BITS-1:0] hits; + logic [PERF_CTR_BITS-1:0] box_tests; + logic [PERF_CTR_BITS-1:0] tri_tests; + logic [PERF_CTR_BITS-1:0] callbacks; + logic [PERF_CTR_BITS-1:0] busy; + } rtu_perf_t; +`endif + typedef struct packed { logic [PERF_CTR_BITS-1:0] idles; logic [PERF_CTR_BITS-1:0] active_warps; @@ -1361,6 +1372,10 @@ package VX_gpu_pkg; om_perf_t om; cache_perf_t ocache; `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + rtu_perf_t rtu; + cache_perf_t rtcache; + `endif } sysmem_perf_t; typedef struct packed { diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index c14be8a0c4..af9b3258aa 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -104,6 +104,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE tex_perf_t socket_tex_perf; cache_perf_t socket_tcache_perf; +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + rtu_perf_t socket_rtu_perf; + cache_perf_t socket_rtcache_perf; `endif sysmem_perf_t sysmem_perf_tmp; always @(*) begin @@ -117,6 +121,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; sysmem_perf_tmp.tex = socket_tex_perf; sysmem_perf_tmp.tcache = socket_tcache_perf; `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + sysmem_perf_tmp.rtu = socket_rtu_perf; + sysmem_perf_tmp.rtcache = socket_rtcache_perf; + `endif end `endif @@ -575,6 +583,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .TAG_WIDTH (RTCACHE_TAG_WIDTH) ) rtcache_bus_if [`VX_CFG_NUM_RTU_CORES * RTCACHE_NUM_REQS] (); +`ifdef PERF_ENABLE + rtu_perf_t [`VX_CFG_NUM_RTU_CORES-1:0] per_core_rtu_perf; +`endif + for (genvar i = 0; i < `VX_CFG_NUM_RTU_CORES; ++i) begin : g_rtu_core VX_rtu_core #( .INSTANCE_ID (`SFORMATF(("%s-rtu%0d", INSTANCE_ID, i))), @@ -586,11 +598,30 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ) rtu_core ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + .perf_rtu (per_core_rtu_perf[i]), + `endif .rtu_bus_if (rtu_bus_if[i]), .cache_bus_if (rtcache_bus_if[i * RTCACHE_NUM_REQS]) ); end +`ifdef PERF_ENABLE + // Every rtu_perf_t field is one PERF_CTR_BITS counter: sum them field-wise. + localparam RTU_PERF_FIELDS = $bits(rtu_perf_t) / PERF_CTR_BITS; + wire [`VX_CFG_NUM_RTU_CORES-1:0][RTU_PERF_FIELDS-1:0][PERF_CTR_BITS-1:0] rtu_perf_w = per_core_rtu_perf; + reg [RTU_PERF_FIELDS-1:0][PERF_CTR_BITS-1:0] rtu_perf_sum; + always @(*) begin + rtu_perf_sum = '0; + for (integer i = 0; i < `VX_CFG_NUM_RTU_CORES; ++i) begin + for (integer f = 0; f < RTU_PERF_FIELDS; ++f) begin + rtu_perf_sum[f] = rtu_perf_sum[f] + rtu_perf_w[i][f]; + end + end + end + assign socket_rtu_perf = rtu_perf_sum; +`endif + VX_mem_bus_if #( .DATA_SIZE (RTCACHE_LINE_SIZE), .TAG_WIDTH (RTCACHE_MEM_TAG_WIDTH) @@ -647,7 +678,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), `ifdef PERF_ENABLE - `UNUSED_PIN (cache_perf), + .cache_perf (socket_rtcache_perf), `endif .core_bus_if (rtcache_flushable_bus_if), .mem_bus_if (rtcache_mem_bus_tmp_if) diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index f09c17bb7e..a47d6a86be 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -573,6 +573,21 @@ import VX_fpu_pkg::*; default:; endcase end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `VX_DCR_MPM_CLASS_RTU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); + `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); + `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); + `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); + default:; + endcase + end `endif default:; endcase diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index 3f018a13e0..932bcaa8a5 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -53,6 +53,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + output rtu_perf_t perf_rtu, +`endif + // window bus (this core is the master) VX_rtu_bus_if.slave rtu_bus_if, @@ -196,6 +200,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( `UNUSED_VAR (sch_busy) +`ifdef PERF_ENABLE + rtu_sched_events_t sch_events; +`endif + VX_rtu_scheduler #( .INSTANCE_ID (INSTANCE_ID), .NUM_SLOTS (NUM_SLOTS), @@ -203,6 +211,9 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ) scheduler ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + .perf_events (sch_events), + `endif .slot_start_valid (ss_valid), .slot_start_slot (ss_slot), .slot_start_mask (ss_mask), @@ -786,45 +797,65 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( end end -// ── RTU occupancy counters ──────────────────────────────────────────────── -`ifdef DBG_RTU_OCC - longint unsigned occ_total, occ_busy, occ_write, occ_cb, occ_fill; - longint unsigned occ_idle_ray_waiting, occ_idle_starved, occ_traces; +`ifdef PERF_ENABLE + localparam LANE_CNT_W = `CLOG2(NUM_LANES + 1); + + // A record's lanes are counted once, when its status word (the last) goes out. + wire perf_rec_done = win_fire && wr_status; + wire perf_term = perf_rec_done && !is_cand[ws]; + wire perf_cand = perf_rec_done && is_cand[ws]; + + wire [NUM_LANES-1:0] ws_hit = sch_hit[32'(ws)*NUM_LANES +: NUM_LANES]; + + // Shader callbacks: any-hit and intersection candidates returned to the warp. + wire [NUM_LANES-1:0] ws_cb; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_perf_cb + assign ws_cb[l] = wr_cb_mask[l] + && ((sch_cbtype[wctx[l]] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_ANYHIT)) + || (sch_cbtype[wctx[l]] == RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_PROC))); + end + + wire [NUM_LANES-1:0] perf_ray_lanes = ss_valid ? ss_mask : '0; + wire [NUM_LANES-1:0] perf_hit_lanes = perf_term ? (req_mask[ws] & ws_hit) : '0; + wire [NUM_LANES-1:0] perf_cb_lanes = perf_cand ? ws_cb : '0; + + wire [LANE_CNT_W-1:0] perf_rays_cnt, perf_hits_cnt, perf_cb_cnt; + `POP_COUNT(perf_rays_cnt, perf_ray_lanes); + `POP_COUNT(perf_hits_cnt, perf_hit_lanes); + `POP_COUNT(perf_cb_cnt, perf_cb_lanes); + + // The per-cycle increments are registered so the counter adders stay off + // the scheduler's and the record walk's timing paths. + rtu_sched_events_t sch_events_r; + reg busy_r; + reg [LANE_CNT_W-1:0] rays_r, hits_r, cb_r; + rtu_perf_t perf_rtu_r; + always @(posedge clk) begin if (reset) begin - occ_total <= 0; occ_busy <= 0; occ_write <= 0; occ_cb <= 0; - occ_fill <= 0; occ_idle_ray_waiting <= 0; occ_idle_starved <= 0; - occ_traces <= 0; + sch_events_r <= '0; + busy_r <= 1'b0; + rays_r <= '0; + hits_r <= '0; + cb_r <= '0; + perf_rtu_r <= '0; end else begin - occ_total <= occ_total + 1; - if (ss_valid) begin - occ_traces <= occ_traces + 1; - end - for (integer s = 0; s < NUM_SLOTS; s = s + 1) begin - case (tstate[s]) - T_BUSY: occ_busy <= occ_busy + 1; - T_WRITE: occ_write <= occ_write + 1; - T_FILL: occ_fill <= occ_fill + 1; - T_CBWAIT, T_CBATTR, T_RESUME, T_RWAIT: occ_cb <= occ_cb + 1; - T_IDLE: begin - if ((| stg_ready) || (fstate == F_RUN)) begin - occ_idle_ray_waiting <= occ_idle_ray_waiting + 1; - end else begin - occ_idle_starved <= occ_idle_starved + 1; - end - end - default:; - endcase - end - end - end - always @(posedge clk) begin - if (!reset && (occ_total % 40000 == 39999)) begin - $display("RTU-OCC @%0d: traces=%0d | busy=%0d write=%0d cb=%0d fill=%0d | IDLE_ray_waiting=%0d IDLE_STARVED=%0d", - occ_total, occ_traces, occ_busy, occ_write, occ_cb, occ_fill, - occ_idle_ray_waiting, occ_idle_starved); + sch_events_r <= sch_events; + busy_r <= ~(& slot_free); + rays_r <= perf_rays_cnt; + hits_r <= perf_hits_cnt; + cb_r <= perf_cb_cnt; + + perf_rtu_r.rays <= perf_rtu_r.rays + PERF_CTR_BITS'(rays_r); + perf_rtu_r.hits <= perf_rtu_r.hits + PERF_CTR_BITS'(hits_r); + perf_rtu_r.box_tests <= perf_rtu_r.box_tests + PERF_CTR_BITS'(sch_events_r.box_test); + perf_rtu_r.tri_tests <= perf_rtu_r.tri_tests + PERF_CTR_BITS'(sch_events_r.tri_test); + perf_rtu_r.callbacks <= perf_rtu_r.callbacks + PERF_CTR_BITS'(cb_r); + perf_rtu_r.busy <= perf_rtu_r.busy + PERF_CTR_BITS'(busy_r); end end + + assign perf_rtu = perf_rtu_r; `endif endmodule diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index 2f31dfe102..872e3df571 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -303,6 +303,13 @@ package VX_rtu_pkg; localparam RTU_WS_WORDS = 32; // rows per slot (power of two for addressing) localparam RTU_WS_WORD_BITS = `CLOG2(RTU_WS_WORDS); + // Per-cycle traversal events the scheduler reports for the performance + // counters: strobes only, the counting lives in the core. + typedef struct packed { + logic box_test; // a ray/AABB test issued + logic tri_test; // a ray/triangle test issued + } rtu_sched_events_t; + `IGNORE_UNUSED_END endpackage diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 46f1676251..dab898a532 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -61,6 +61,10 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( input wire clk, input wire reset, +`ifdef PERF_ENABLE + output rtu_sched_events_t perf_events, +`endif + // ── slot launch (from the core's fill engine) ───────────────────── // slot_start arms the slot (mask + warp-uniform ray scalars) before any of // its rays land; each lane's ray then arrives on ray_wr, which is also that @@ -1908,6 +1912,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( ("%t: rtu walk deeper than %0d levels", $time, (1 << LVLW) - 1)) assign path_wlvl = path_wlvl_r; assign path_wdata = path_wdata_r; + +`ifdef PERF_ENABLE + assign perf_events.box_test = box_feed || box_feed_raw; + assign perf_events.tri_test = tri_feed; +`endif assign mem_req_valid = x_valid && mem_issue; assign mem_req_addr = structaddr_q + (ADDRW'(mem_fslot) << RTU_LINE_SEL_BITS); assign mem_req_tag = sel_q; diff --git a/hw/unittest/rtu_core/VX_rtu_core_top.sv b/hw/unittest/rtu_core/VX_rtu_core_top.sv index f4405346dd..f023e8f316 100644 --- a/hw/unittest/rtu_core/VX_rtu_core_top.sv +++ b/hw/unittest/rtu_core/VX_rtu_core_top.sv @@ -123,6 +123,9 @@ module VX_rtu_core_top import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ) rtu_core ( .clk (clk), .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rtu), + `endif .rtu_bus_if (rtu_bus_if), .cache_bus_if (cache_bus_if) ); diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 550ec14fca..9291bf8a86 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -305,6 +305,21 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { CSR_READ_64(VX_CSR_MPM_OCACHE_MSHR_ST,cluster_perf.ocache.mshr_stalls); } } break; + #endif + #ifdef VX_CFG_EXT_RTU_ENABLE + case VX_DCR_MPM_CLASS_RTU: { + auto socket_perf = core_->socket()->perf_stats(); + switch (addr) { + CSR_READ_64(VX_CSR_MPM_RTU_RAYS, socket_perf.rtu.rays); + CSR_READ_64(VX_CSR_MPM_RTU_HITS, socket_perf.rtu.rays_hit); + CSR_READ_64(VX_CSR_MPM_RTU_BOX_TESTS, socket_perf.rtu.bvh_box_tests); + CSR_READ_64(VX_CSR_MPM_RTU_TRI_TESTS, socket_perf.rtu.bvh_tri_tests); + CSR_READ_64(VX_CSR_MPM_RTU_CALLBACKS, (socket_perf.rtu.ahs_callbacks + socket_perf.rtu.is_callbacks)); + CSR_READ_64(VX_CSR_MPM_RTU_BUSY, socket_perf.rtu.rtu_busy_ticks); + CSR_READ_64(VX_CSR_MPM_RTCACHE_READS, socket_perf.rtcache.reads); + CSR_READ_64(VX_CSR_MPM_RTCACHE_MISS_R, socket_perf.rtcache.read_misses); + } + } break; #endif default: std::cerr << "Error: invalid MPM CLASS: value=" << perf_class << std::endl; diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index b8f53f321e..3ad2f0a7e8 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -184,8 +184,6 @@ class ReformationEngine { port.send(rsp); warp_cb_inflight_[anchor_warp] = true; ++perf_.reformation_yields; - // cb_type is uniform within a batched yield (grouped by sbt_idx), so one - // sample names the per-yield-type counter. for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (!(cb_mask & (1u << t))) continue; switch (rsp.cb_type[t]) { @@ -195,7 +193,6 @@ class ReformationEngine { case VX_RT_CB_TYPE_MISS: ++perf_.miss_callbacks; break; default: break; } - break; } DT(3, "rtu-core reform cb_yield: warp=" << anchor_warp << ", sbt=" << anchor_sbt @@ -255,7 +252,7 @@ class RtuCore::Impl { const auto& p = perf_stats_; std::fprintf(stderr, "[rtu-stats] slots=%u ctx=%u merge=%u\n" - "[rtu-stats] rays_issued=%llu rays_hit=%llu rays_miss=%llu\n" + "[rtu-stats] traces=%llu rays=%llu rays_hit=%llu rays_miss=%llu\n" "[rtu-stats] mem_reads=%llu fetches_merged=%llu mshr_full=%llu\n" "[rtu-stats] bvh_nodes=%llu bvh_leaves=%llu box_tests=%llu tri_tests=%llu\n" "[rtu-stats] busy_ticks=%llu front_end_ticks=%llu mem_issue_ticks=%llu\n" @@ -263,7 +260,8 @@ class RtuCore::Impl { "[rtu-stats] cb_ahs=%llu cb_chs=%llu cb_miss=%llu cb_is=%llu reform=%llu\n", uint32_t(VX_CFG_RTU_NUM_SLOTS), uint32_t(VX_CFG_RTU_NUM_CTX), uint32_t(VX_CFG_RTU_MERGE_DEPTH), - (unsigned long long)p.rays_issued, + (unsigned long long)p.traces, + (unsigned long long)p.rays, (unsigned long long)p.rays_hit, (unsigned long long)p.rays_miss, (unsigned long long)p.mem_reads, @@ -291,7 +289,8 @@ class RtuCore::Impl { for (auto& cx : contexts_) cx = Context{}; mem_engine_.reset(); reform_.reset(); - perf_stats_ = RtuCore::PerfStats(); + // perf_stats_ survives: the MPM counters run from device reset, and every + // kernel launch re-enters reset(). last_signature_ = 0; rr_exec_ = 0; step_cycles_left_ = 0; @@ -359,7 +358,8 @@ class RtuCore::Impl { } if (raylog::enabled()) raylog::on_accept(this, idx, s.req); ch.pop(); - ++perf_stats_.rays_issued; + ++perf_stats_.traces; + perf_stats_.rays += __builtin_popcount(s.req.tmask_bits); DT(3, "rtu-core accept: tag=" << s.req.tag << ", slot=" << idx); rr_req_ = (cid + 1) % chs.size(); return; @@ -569,13 +569,12 @@ class RtuCore::Impl { + inst * kRtuImageStatesPerInst; // Behind the last test fed, the PE pipeline drains while the context is // PARKED — box and tri results both carry their context id, so waiting for - // one costs the shared front end nothing. A trail restart re-descends every - // subtree the short stack had to evict. + // one costs the shared front end nothing. A trail restart's re-descent is + // already in the node and box counts above. uint32_t lat = 0; if (box) lat = std::max(lat, BoxPe::pipe_depth()); if (tri) lat = std::max(lat, TriPe::pipe_depth()); lat += kRtuXformLatency * inst; - lat += VX_CFG_RTU_STACK_DEPTH * restart; cx.pe_lat = lat; perf_stats_.walker_cycles_total += cx.fsm_states * kRtuPhasesPerStep + cx.img_states * kRtuPhasesImageAdd + lat; @@ -723,8 +722,18 @@ class RtuCore::Impl { void sample_occupancy() { uint32_t slots_used = 0; + bool busy = false; for (uint32_t i = 0; i < pool_.size(); ++i) { - if (pool_.at(i).in_use) ++slots_used; + const Slot& s = pool_.at(i); + if (s.in_use) ++slots_used; + // A slot is reserved before its rays arrive and stays emitted until the + // warp consumes its record; it holds no trace in either state. + if (s.in_use && s.state != SlotState::RESERVED && s.state != SlotState::EMITTED) { + busy = true; + } + } + if (busy) { + ++perf_stats_.rtu_busy_ticks; } uint32_t bound = 0, active = 0; for (const auto& cx : contexts_) { @@ -738,7 +747,6 @@ class RtuCore::Impl { perf_stats_.slot_busy_ticks += slots_used; perf_stats_.ctx_bound_ticks += bound; perf_stats_.ctx_active_ticks += active; - if (slots_used != 0) ++perf_stats_.rtu_busy_ticks; } const RtuCore::PerfStats& perf_stats() const { return perf_stats_; } diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 8e84ba11cb..cb72cd528a 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -542,7 +542,8 @@ struct QueueEntry { // ════════════════════════════════════════════════════════════════════ struct PerfStats { - uint64_t rays_issued = 0; + uint64_t traces = 0; // warp-level traces accepted + uint64_t rays = 0; // active lanes of those traces uint64_t rays_hit = 0; uint64_t rays_miss = 0; uint64_t mem_reads = 0; @@ -552,12 +553,10 @@ struct PerfStats { uint64_t bvh_instance_descents = 0; uint64_t bvh_box_tests = 0; uint64_t bvh_tri_tests = 0; - // Short-stack overflow events: pushes past the VX_CFG_RTU_STACK_DEPTH HW - // stack. SimX keeps an unbounded stack (never misses a hit); each overflow - // entry is one the HW must re-descend for via trail-based restart, charged - // in the cost model. + // Trail restarts of the VX_CFG_RTU_STACK_DEPTH short stack. The node, box and + // descent counts above include the restarted walks' re-descents. uint64_t bvh_stack_restarts = 0; - // Callback-pipeline counters. + // Callback-pipeline counters: candidate lanes returned to the warp, by type. uint64_t ahs_callbacks = 0; uint64_t chs_callbacks = 0; uint64_t miss_callbacks = 0; @@ -586,14 +585,15 @@ struct PerfStats { uint64_t ctx_bound_ticks = 0; uint64_t ctx_active_ticks = 0; uint64_t slot_busy_ticks = 0; - uint64_t rtu_busy_ticks = 0; // ticks with any slot in use + uint64_t rtu_busy_ticks = 0; // ticks with any slot holding a trace // SIMD-PE cycle accounting: pipeline cycles charged across every context's // traversal, and ticks where at least one context was draining them. uint64_t walker_cycles_total = 0; uint64_t walker_busy_ticks = 0; PerfStats& operator+=(const PerfStats& rhs) { - rays_issued += rhs.rays_issued; + traces += rhs.traces; + rays += rhs.rays; rays_hit += rhs.rays_hit; rays_miss += rhs.rays_miss; mem_reads += rhs.mem_reads; diff --git a/sim/simx/rtu/rtu_walker.cpp b/sim/simx/rtu/rtu_walker.cpp index 3164e9f9d7..eaedcc9c8e 100644 --- a/sim/simx/rtu/rtu_walker.cpp +++ b/sim/simx/rtu/rtu_walker.cpp @@ -14,6 +14,7 @@ #include "rtu_walker.h" #include +#include #include #include #include @@ -124,21 +125,55 @@ inline uint32_t hit_facing_bit(bool back_facing, uint32_t inst_flags) { return back_facing ? VX_RT_HIT_BACK_FACING : 0u; } +// The hardware's per-ray traversal stack: VX_CFG_RTU_STACK_DEPTH entries shared +// by the TLAS and every BLAS below it. A walk visits the tree depth first, +// nearer child first, so every node has a rank path (its child rank at each +// level, an instance's index within its leaf) and the walk proceeds in +// ascending rank-path order. A child that does not fit on the stack is dropped; +// the walk then only descends, so when it would next pop it restarts from the +// root instead, following the rank path to the nearest dropped child: every +// node before it has been visited. A tightened best_t only culls a suffix of a +// node's ordered children, so the recorded ranks stay valid across restarts. +struct ShortStack { + static constexpr uint32_t kLevels = 64; + struct Entry { + uint32_t lvl; + uint32_t rank; + uint32_t off; + }; + std::vector entries; + std::array path{}; // rank taken at each level + bool ovf = false; // a child was dropped since the walk (re)started + uint32_t nd_lvl = 0; // the nearest dropped child + uint32_t nd_rank = 0; + bool follow = false; // restarting: descend along the rank path + uint32_t trl_lvl = 0; // ... down to this dropped child + uint32_t trl_rank = 0; + bool restart = false; // the walk must unwind and start over + uint64_t visits = 0; + + void set_path(uint32_t lvl, uint32_t rank) { + if (lvl < kLevels) path[lvl] = rank; + } + uint32_t get_path(uint32_t lvl) const { + return (lvl < kLevels) ? path[lvl] : 0; + } +}; + // Depth-first walker for one BVH sub-tree under the supplied (object-space) -// ray. Recurses on LeafInst so each instance's BLAS gets walked with its -// transformed ray. ctx accumulates hits/yields across the whole call tree. -// -// Returns as soon as sv.miss is raised: the caller discards the partial state -// and replays the walk once the missing line has arrived. +// ray, rooted at level root_lvl. Recurses on LeafInst so each instance's BLAS +// gets walked with its transformed ray on the same short stack. ctx accumulates +// hits/yields across the whole call tree. // -// The functional traversal stack is unbounded so the oracle never misses a hit; -// the HW short-stack (VX_CFG_RTU_STACK_DEPTH) overflow is counted as a -// trail-based RESTART and charged in the cost model. +// Returns as soon as sv.miss is raised (the caller discards the partial state +// and replays the walk once the missing line has arrived) or ss.restart is set +// (the caller restarts the walk from the scene root). void walk_bvh4_subtree(SceneView& sv, const float ro[3], const float rd[3], - uint32_t root_off, uint32_t instance_id, - uint32_t custom_id, uint32_t inst_flags, - WalkCtx& ctx, PerfStats& perf) { + uint32_t root_off, uint32_t root_lvl, + uint32_t instance_id, uint32_t custom_id, + uint32_t inst_flags, WalkCtx& ctx, ShortStack& ss, + PerfStats& perf) { // The ray setup's reciprocals, once per (object-space) ray. const float inv[3] = { ray_recip(rd[0]), ray_recip(rd[1]), ray_recip(rd[2]) }; @@ -263,12 +298,14 @@ void walk_bvh4_subtree(SceneView& sv, } }; - auto visit_leaf_inst = [&](uint32_t leaf_off, uint32_t count) { + auto visit_leaf_inst = [&](uint32_t leaf_off, uint32_t count, uint32_t lvl) { uint8_t hdr_buf[kVxBvhLeafHeaderBytes]; read_scene_bytes(sv, leaf_off, sizeof(hdr_buf), hdr_buf); if (sv.miss) return; uint32_t insts_off = leaf_off + kVxBvhLeafHeaderBytes; - for (uint32_t i = 0; i < count; ++i) { + // A restart following its path resumes at the instance it recorded. + const uint32_t first = ss.follow ? ss.get_path(lvl + 1) : 0; + for (uint32_t i = first; i < count; ++i) { uint8_t inst_buf[kVxBvhInstanceStride]; read_scene_bytes(sv, insts_off + i * kVxBvhInstanceStride, kVxBvhInstanceStride, inst_buf); @@ -278,41 +315,41 @@ void walk_bvh4_subtree(SceneView& sv, // Vulkan instanceCullMask: skip the instance entirely if its mask byte // and the ray's cull_mask have no bits in common. Both default to 0xff in // the no-culling path. - if ((inst->cull_mask & ctx.ray_cull_mask & 0xffu) == 0) continue; + if ((inst->cull_mask & ctx.ray_cull_mask & 0xffu) == 0) { + ss.follow = false; + continue; + } // VkGeometryInstanceFlagBits packed into cull_mask bits 15..8. uint32_t inst_flags2 = (inst->cull_mask >> kRtuInstanceFlagsShift) & kRtuInstanceFlagsMask; float obj_ro[3], obj_rd[3]; world_to_object_ray(inst->xform, ro, rd, obj_ro, obj_rd); ++perf.bvh_instance_descents; + // The BLAS root sits at the instance level, ranked by its index. + ss.set_path(lvl + 1, i); walk_bvh4_subtree(sv, obj_ro, obj_rd, - inst->blas_root_byte_offset, + inst->blas_root_byte_offset, lvl + 1, inst->instance_id, inst->custom_id, inst_flags2, - ctx, perf); - if (sv.miss) return; + ctx, ss, perf); + if (sv.miss || ss.restart || ctx.terminated) return; + ss.follow = false; } }; - // SimX is the correctness oracle: keep an UNBOUNDED traversal stack so deep - // sub-trees are never dropped. The HW short-stack is only - // VX_CFG_RTU_STACK_DEPTH deep and trail-restarts to re-find subtrees it had - // to evict — it visits the same leaves, just at extra cost. - std::vector stack; - stack.reserve(VX_CFG_RTU_STACK_DEPTH); - uint32_t current = root_off; - bool have_current = true; - // Safety backstop: a malformed/cyclic acceleration structure (a child offset - // pointing back at an ancestor) would otherwise descend forever with the - // unbounded stack. The ceiling is far above any well-formed scene's node - // count, so it never truncates a legitimate walk. + // pointing back at an ancestor) would otherwise descend forever. The ceiling + // is far above any well-formed scene's node count, so it never truncates a + // legitimate walk. constexpr uint64_t kMaxNodeVisits = 1ull << 22; - uint64_t node_visits = 0; - while (have_current) { - if (ctx.terminated) break; - if (++node_visits > kMaxNodeVisits) break; + // The stack depth this sub-tree started from: popping back to it ends a BLAS. + const size_t stack_floor = ss.entries.size(); + uint32_t current = root_off; + uint32_t lvl = root_lvl; + + while (!ctx.terminated) { + if (++ss.visits > kMaxNodeVisits) return; uint8_t kind_buf[4]; read_scene_bytes(sv, current, sizeof(kind_buf), kind_buf); if (sv.miss) return; @@ -329,8 +366,8 @@ void walk_bvh4_subtree(SceneView& sv, } } else if (kind == kVxBvhKindLeafInst) { ++perf.bvh_leaves_fetched; - visit_leaf_inst(current, count); - if (sv.miss) return; + visit_leaf_inst(current, count, lvl); + if (sv.miss || ss.restart) return; } else if (kind == kVxBvhKindLeafProc) { ++perf.bvh_leaves_fetched; // SKIP_AABBS: symmetric gate with SKIP_TRIANGLES. Otherwise ray-test each @@ -388,27 +425,53 @@ void walk_bvh4_subtree(SceneView& sv, } hits[j] = h; } - if (hit_count > 0) { - for (uint32_t i = hit_count; i-- > 1; ) { - // Each push past the HW short-stack depth is one subtree the HW would - // have to re-descend for via restart — count it for the cost model, - // but keep it (unbounded) so the functional walk never misses a hit. - if (stack.size() >= VX_CFG_RTU_STACK_DEPTH) - ++perf.bvh_stack_restarts; - stack.push_back(hits[i].offset); + + // A restart descends into the child its path names; nearer children + // were visited before. Culled by now, that child and every farther one + // are done. + uint32_t dsc = 0; + bool descend = (hit_count != 0); + if (ss.follow) { + dsc = (lvl + 1 == ss.trl_lvl) ? ss.trl_rank : ss.get_path(lvl + 1); + if (hit_count <= dsc) { + ss.follow = false; + descend = false; + } + } + if (descend) { + // Farther children first, so the last one dropped is the nearest. + for (uint32_t i = hit_count; i-- > dsc + 1; ) { + if (ss.entries.size() < VX_CFG_RTU_STACK_DEPTH) { + ss.entries.push_back({lvl + 1, i, hits[i].offset}); + } else { + ss.ovf = true; + ss.nd_lvl = lvl + 1; + ss.nd_rank = i; + } + } + current = hits[dsc].offset; + ++lvl; + ss.set_path(lvl, dsc); + if (ss.follow && lvl == ss.trl_lvl) { + ss.follow = false; } - current = hits[0].offset; - have_current = true; continue; } } - if (stack.empty()) { - have_current = false; - } else { - current = stack.back(); - stack.pop_back(); + if (ctx.terminated) return; + if (ss.ovf) { + ++perf.bvh_stack_restarts; + ss.restart = true; + return; } + if (ss.entries.size() == stack_floor) return; + ss.follow = false; + const ShortStack::Entry e = ss.entries.back(); + ss.entries.pop_back(); + current = e.off; + lvl = e.lvl; + ss.set_path(e.lvl, e.rank); } } @@ -711,8 +774,19 @@ WalkResult Bvh4Walker::walk_lane(const RtuReq& req, uint32_t t, SceneView& sv, WalkCtx ctx = init_ctx(req, t, ro, rd, l); // Top-level (non-instanced) triangles carry no instance flags. - walk_bvh4_subtree(sv, ro, rd, root_off, 0, 0, 0, ctx, perf); - if (sv.miss) return {true, false}; + ShortStack ss; + ss.entries.reserve(VX_CFG_RTU_STACK_DEPTH); + for (;;) { + walk_bvh4_subtree(sv, ro, rd, root_off, 0, 0, 0, 0, ctx, ss, perf); + if (sv.miss) return {true, false}; + if (!ss.restart) break; + ss.entries.clear(); + ss.restart = false; + ss.ovf = false; + ss.follow = true; + ss.trl_lvl = ss.nd_lvl; + ss.trl_rank = ss.nd_rank; + } return {false, emit_lane_result(req, l, t, ctx)}; } diff --git a/sw/runtime/common/perf.cpp b/sw/runtime/common/perf.cpp index cd357ab8fa..8489a3a59d 100644 --- a/sw/runtime/common/perf.cpp +++ b/sw/runtime/common/perf.cpp @@ -161,6 +161,75 @@ struct CacheCounters { uint64_t mshr_st = 0; }; +struct RtuCounters { + uint64_t rays = 0; + uint64_t hits = 0; + uint64_t box_tests = 0; + uint64_t tri_tests = 0; + uint64_t callbacks = 0; + uint64_t busy = 0; + CacheCounters rtcache; + + RtuCounters& operator+=(const RtuCounters& rhs) { + rays += rhs.rays; + hits += rhs.hits; + box_tests += rhs.box_tests; + tri_tests += rhs.tri_tests; + callbacks += rhs.callbacks; + busy += rhs.busy; + rtcache.reads += rhs.rtcache.reads; + rtcache.miss_r += rhs.rtcache.miss_r; + return *this; + } +}; + +static vx_result_t query_rtu_counters(vx_device_h hdevice, uint32_t mpm_class, + uint32_t core_id, RtuCounters* c) { + const struct { + uint32_t addr; + uint64_t* value; + } fields[] = { + {VX_CSR_MPM_RTU_RAYS, &c->rays}, + {VX_CSR_MPM_RTU_HITS, &c->hits}, + {VX_CSR_MPM_RTU_BOX_TESTS, &c->box_tests}, + {VX_CSR_MPM_RTU_TRI_TESTS, &c->tri_tests}, + {VX_CSR_MPM_RTU_CALLBACKS, &c->callbacks}, + {VX_CSR_MPM_RTU_BUSY, &c->busy}, + {VX_CSR_MPM_RTCACHE_READS, &c->rtcache.reads}, + {VX_CSR_MPM_RTCACHE_MISS_R, &c->rtcache.miss_r}, + }; + for (const auto& f : fields) { + CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, f.addr, core_id, f.value), { return err; }); + } + return VX_SUCCESS; +} + +// Raw counters first (name=value), derived metrics in parentheses. `units` is +// the number of RTU sockets the counters were summed over, so the utilization +// stays a per-RTU average. +static void print_rtu_counters(FILE* stream, int core_id, const RtuCounters& c, + uint64_t cycles, uint64_t units) { + const double rays = (double)c.rays; + const double unit_cycles = (double)cycles * (double)units; + char buf[512]; + std::snprintf(buf, sizeof(buf), + "rtu: rays=%" PRIu64 ", hits=%" PRIu64 " (rate=%d%%), box_tests=%" PRIu64 " (%.1f/ray)" + ", tri_tests=%" PRIu64 " (%.1f/ray), callbacks=%" PRIu64 ", busy=%" PRIu64 " (util=%d%%)" + "; rtcache: reqs=%" PRIu64 " (%.1f/ray), miss=%" PRIu64 " (hit=%d%%)", + c.rays, c.hits, calc_percent(c.hits, c.rays), + c.box_tests, safe_div((double)c.box_tests, rays), + c.tri_tests, safe_div((double)c.tri_tests, rays), + c.callbacks, + c.busy, (int)std::lround(safe_div((double)c.busy, unit_cycles) * 100.0), + c.rtcache.reads, safe_div((double)c.rtcache.reads, rays), + c.rtcache.miss_r, calc_ratio(c.rtcache.miss_r, c.rtcache.reads)); + if (core_id >= 0) { + perf_print_core(stream, core_id, "%s", buf); + } else { + perf_print(stream, "%s", buf); + } +} + struct Metric { const char *label; int value; @@ -276,6 +345,7 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { const bool tex_en = (isa_flags & VX_ISA_EXT_TEX) != 0; const bool raster_en = (isa_flags & VX_ISA_EXT_RASTER) != 0; const bool om_en = (isa_flags & VX_ISA_EXT_OM) != 0; + const bool rtu_en = (isa_flags & VX_ISA_EXT_RTU) != 0; std::vector cores(num_cores); uint64_t total_instrs = 0; @@ -785,6 +855,28 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { ocache_tot.mshr_st, calc_utility(ocache_tot.miss_r + ocache_tot.miss_w, ocache_tot.mshr_st)); } break; + case VX_DCR_MPM_CLASS_RTU: { + if (!rtu_en) { + perf_print(stream, "RTU not supported"); + break; + } + // Socket-scoped counters: every core of a socket reads the same values. + const uint64_t num_sockets = (num_cores + socket_size - 1) / socket_size; + RtuCounters tot; + for (uint32_t s = 0; s < num_sockets; ++s) { + uint32_t rep_core = s * (uint32_t)socket_size; + if (rep_core >= num_cores) + break; + RtuCounters c; + CHECK_ERR(query_rtu_counters(hdevice, mpm_class, rep_core, &c), { return err; }); + if (num_sockets > 1) { + print_rtu_counters(stream, rep_core, c, cores[rep_core].cycles, 1); + } + tot += c; + } + print_rtu_counters(stream, -1, tot, max_cycles, num_sockets); + } break; + default: fprintf(stream, "Error: invalid profiling class: %d)", mpm_class); return VX_ERR_INVALID_VALUE; diff --git a/tests/raytracing/rt_bvh_multinode/main.cpp b/tests/raytracing/rt_bvh_multinode/main.cpp index 9364557cd7..851ea7dbbb 100644 --- a/tests/raytracing/rt_bvh_multinode/main.cpp +++ b/tests/raytracing/rt_bvh_multinode/main.cpp @@ -62,6 +62,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp index cf0dd113c7..84662c9bb5 100644 --- a/tests/raytracing/rt_replay/main.cpp +++ b/tests/raytracing/rt_replay/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); device = nullptr; } diff --git a/tests/raytracing/rt_smoke/main.cpp b/tests/raytracing/rt_smoke/main.cpp index fc6bf196e3..6c020773b4 100644 --- a/tests/raytracing/rt_smoke/main.cpp +++ b/tests/raytracing/rt_smoke/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs/main.cpp b/tests/raytracing/rt_smoke_ahs/main.cpp index 5ce507f215..4fef4c6724 100644 --- a/tests/raytracing/rt_smoke_ahs/main.cpp +++ b/tests/raytracing/rt_smoke_ahs/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_bvh/main.cpp b/tests/raytracing/rt_smoke_ahs_bvh/main.cpp index b25a1db116..0e83e86bdf 100644 --- a/tests/raytracing/rt_smoke_ahs_bvh/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_bvh/main.cpp @@ -80,6 +80,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_custom/main.cpp b/tests/raytracing/rt_smoke_ahs_custom/main.cpp index 0f04741653..9f60a8f9ae 100644 --- a/tests/raytracing/rt_smoke_ahs_custom/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_custom/main.cpp @@ -68,6 +68,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_geom/main.cpp b/tests/raytracing/rt_smoke_ahs_geom/main.cpp index 01703d4331..c5417af0de 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/main.cpp @@ -69,6 +69,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_mixed/main.cpp b/tests/raytracing/rt_smoke_ahs_mixed/main.cpp index e0839e2322..106f892809 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_mixed/main.cpp @@ -71,6 +71,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_ahs_multi/main.cpp b/tests/raytracing/rt_smoke_ahs_multi/main.cpp index ab7176e2c4..b36ef03195 100644 --- a/tests/raytracing/rt_smoke_ahs_multi/main.cpp +++ b/tests/raytracing/rt_smoke_ahs_multi/main.cpp @@ -81,6 +81,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_async_batch/main.cpp b/tests/raytracing/rt_smoke_async_batch/main.cpp index 4e4117bf90..3eaeea5346 100644 --- a/tests/raytracing/rt_smoke_async_batch/main.cpp +++ b/tests/raytracing/rt_smoke_async_batch/main.cpp @@ -73,6 +73,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh6/main.cpp b/tests/raytracing/rt_smoke_bvh6/main.cpp index 62d06a0788..9bf596ca66 100644 --- a/tests/raytracing/rt_smoke_bvh6/main.cpp +++ b/tests/raytracing/rt_smoke_bvh6/main.cpp @@ -65,6 +65,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_basic/main.cpp b/tests/raytracing/rt_smoke_bvh_basic/main.cpp index 4fbca70264..5cd9cce4bc 100644 --- a/tests/raytracing/rt_smoke_bvh_basic/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_basic/main.cpp @@ -65,6 +65,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp index 73ee543537..a0d51cdf0f 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/main.cpp @@ -76,6 +76,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp b/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp index 5d5b835dce..d480019dee 100644 --- a/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp +++ b/tests/raytracing/rt_smoke_bvh_multilevel/main.cpp @@ -74,6 +74,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_chs/main.cpp b/tests/raytracing/rt_smoke_chs/main.cpp index 6e3b95c3ab..ad82c921ef 100644 --- a/tests/raytracing/rt_smoke_chs/main.cpp +++ b/tests/raytracing/rt_smoke_chs/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_cull_back/main.cpp b/tests/raytracing/rt_smoke_cull_back/main.cpp index 64f9b18677..83e03a524b 100644 --- a/tests/raytracing/rt_smoke_cull_back/main.cpp +++ b/tests/raytracing/rt_smoke_cull_back/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_cull_mask/main.cpp b/tests/raytracing/rt_smoke_cull_mask/main.cpp index 0664fdc290..3ac6722370 100644 --- a/tests/raytracing/rt_smoke_cull_mask/main.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_deep_stack/main.cpp b/tests/raytracing/rt_smoke_deep_stack/main.cpp index 77e900cbe9..205ffae302 100644 --- a/tests/raytracing/rt_smoke_deep_stack/main.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/main.cpp @@ -58,6 +58,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_deep_tlas/main.cpp b/tests/raytracing/rt_smoke_deep_tlas/main.cpp index e3eaf87fd4..a6281fe2e5 100644 --- a/tests/raytracing/rt_smoke_deep_tlas/main.cpp +++ b/tests/raytracing/rt_smoke_deep_tlas/main.cpp @@ -70,6 +70,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_fat_leaf/main.cpp b/tests/raytracing/rt_smoke_fat_leaf/main.cpp index 505cbb6a73..e13024778a 100644 --- a/tests/raytracing/rt_smoke_fat_leaf/main.cpp +++ b/tests/raytracing/rt_smoke_fat_leaf/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_host_cfg/main.cpp b/tests/raytracing/rt_smoke_host_cfg/main.cpp index 36f9e27a75..fa08bc677a 100644 --- a/tests/raytracing/rt_smoke_host_cfg/main.cpp +++ b/tests/raytracing/rt_smoke_host_cfg/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_inst_flags/main.cpp b/tests/raytracing/rt_smoke_inst_flags/main.cpp index 5d19e2b97c..40efd9cac5 100644 --- a/tests/raytracing/rt_smoke_inst_flags/main.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/main.cpp @@ -81,6 +81,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_is/main.cpp b/tests/raytracing/rt_smoke_is/main.cpp index f5b9f26c82..f82a0ae949 100644 --- a/tests/raytracing/rt_smoke_is/main.cpp +++ b/tests/raytracing/rt_smoke_is/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_miss/main.cpp b/tests/raytracing/rt_smoke_miss/main.cpp index a0a4774c6a..2d41da2c3f 100644 --- a/tests/raytracing/rt_smoke_miss/main.cpp +++ b/tests/raytracing/rt_smoke_miss/main.cpp @@ -52,6 +52,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_multitri/main.cpp b/tests/raytracing/rt_smoke_multitri/main.cpp index a3e8bc06ce..2efc8025c3 100644 --- a/tests/raytracing/rt_smoke_multitri/main.cpp +++ b/tests/raytracing/rt_smoke_multitri/main.cpp @@ -76,6 +76,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_partial_mask/main.cpp b/tests/raytracing/rt_smoke_partial_mask/main.cpp index e429da77dd..b65a00e2ba 100644 --- a/tests/raytracing/rt_smoke_partial_mask/main.cpp +++ b/tests/raytracing/rt_smoke_partial_mask/main.cpp @@ -51,6 +51,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_proc/main.cpp b/tests/raytracing/rt_smoke_proc/main.cpp index d5281bdd33..2e656172f8 100644 --- a/tests/raytracing/rt_smoke_proc/main.cpp +++ b/tests/raytracing/rt_smoke_proc/main.cpp @@ -61,6 +61,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_proc_inst/main.cpp b/tests/raytracing/rt_smoke_proc_inst/main.cpp index 539a8d3aa1..b8c0967c3a 100644 --- a/tests/raytracing/rt_smoke_proc_inst/main.cpp +++ b/tests/raytracing/rt_smoke_proc_inst/main.cpp @@ -67,6 +67,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_recursive/main.cpp b/tests/raytracing/rt_smoke_recursive/main.cpp index e9bfd26120..6be568cb6d 100644 --- a/tests/raytracing/rt_smoke_recursive/main.cpp +++ b/tests/raytracing/rt_smoke_recursive/main.cpp @@ -54,6 +54,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform/main.cpp b/tests/raytracing/rt_smoke_reform/main.cpp index fa2a191e63..29f32fabd0 100644 --- a/tests/raytracing/rt_smoke_reform/main.cpp +++ b/tests/raytracing/rt_smoke_reform/main.cpp @@ -75,6 +75,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform_mw/main.cpp b/tests/raytracing/rt_smoke_reform_mw/main.cpp index 27542a24a0..5d3984ce09 100644 --- a/tests/raytracing/rt_smoke_reform_mw/main.cpp +++ b/tests/raytracing/rt_smoke_reform_mw/main.cpp @@ -80,6 +80,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_reform_sbt/main.cpp b/tests/raytracing/rt_smoke_reform_sbt/main.cpp index 985f95fdbd..b3b4063b52 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/main.cpp +++ b/tests/raytracing/rt_smoke_reform_sbt/main.cpp @@ -82,6 +82,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_sbt/main.cpp b/tests/raytracing/rt_smoke_sbt/main.cpp index c87618f5da..d884f5c775 100644 --- a/tests/raytracing/rt_smoke_sbt/main.cpp +++ b/tests/raytracing/rt_smoke_sbt/main.cpp @@ -54,6 +54,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_shadow/main.cpp b/tests/raytracing/rt_smoke_shadow/main.cpp index 1f953ce076..f4e8926912 100644 --- a/tests/raytracing/rt_smoke_shadow/main.cpp +++ b/tests/raytracing/rt_smoke_shadow/main.cpp @@ -63,6 +63,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_tlas/main.cpp b/tests/raytracing/rt_smoke_tlas/main.cpp index aa7d38a850..f19f87b686 100644 --- a/tests/raytracing/rt_smoke_tlas/main.cpp +++ b/tests/raytracing/rt_smoke_tlas/main.cpp @@ -50,6 +50,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } diff --git a/tests/raytracing/rt_smoke_tlas_builder/main.cpp b/tests/raytracing/rt_smoke_tlas_builder/main.cpp index 4e8e48bded..ff0f249bc8 100644 --- a/tests/raytracing/rt_smoke_tlas_builder/main.cpp +++ b/tests/raytracing/rt_smoke_tlas_builder/main.cpp @@ -57,6 +57,7 @@ void cleanup() { if (kernel) vx_kernel_release(kernel); if (module_) vx_module_release(module_); if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); vx_device_release(device); } } From 180dce199accfd1e880388c6f7eddb360622394f Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 21:10:03 -0700 Subject: [PATCH 5/6] kernel: align .tdata so every hart's TLS block stays aligned When a kernel has .tbss but no .tdata, ld drops the empty .tdata and the TLS image starts at the aligned .tbss, but __tdata_start stays at the unaligned location counter. __tbss_offset then picks up 1-7 bytes of padding and the per-hart stride __tls_block_size (e.g. 7, 11, 12) is no longer a multiple of the word size, so every hart past the first gets a misaligned tp and __init_tls zeroes .tbss at the wrong offset. A kernel with one thread_local in .tbss and a 1-byte .rodata pad trapped with "misaligned memory access" on rv32 and rv64; it now passes. Kernels without TLS link to the same section layout. Co-Authored-By: Claude Opus 5.5 --- sw/kernel/scripts/link32.ld | 8 +++++--- sw/kernel/scripts/link64.ld | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sw/kernel/scripts/link32.ld b/sw/kernel/scripts/link32.ld index b288bcff3b..433ed50de3 100644 --- a/sw/kernel/scripts/link32.ld +++ b/sw/kernel/scripts/link32.ld @@ -101,7 +101,7 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + .tdata : ALIGN(4) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) @@ -119,8 +119,10 @@ SECTIONS /* Per-hart stride for the TLS image, which spans .tdata + .tbss. vx_start.S places each hart's TLS block this far apart; using __tbss_size alone made adjacent harts' images overlap by - __tbss_offset bytes. __tbss_offset and __tbss_size are both - section-aligned, so their sum keeps the image alignment. */ + __tbss_offset bytes. .tdata is aligned like .tbss: when it is empty + the linker drops it and the TLS image starts at .tbss, so an unaligned + __tdata_start would skew __tbss_offset and the stride off the 4-byte + boundary that every hart's tp must keep. */ PROVIDE(__tls_block_size = __tbss_offset + __tbss_size); .preinit_array : { diff --git a/sw/kernel/scripts/link64.ld b/sw/kernel/scripts/link64.ld index aa6b87ff9c..2c92423941 100644 --- a/sw/kernel/scripts/link64.ld +++ b/sw/kernel/scripts/link64.ld @@ -97,7 +97,7 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + .tdata : ALIGN(8) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) @@ -115,8 +115,10 @@ SECTIONS /* Per-hart stride for the TLS image, which spans .tdata + .tbss. vx_start.S places each hart's TLS block this far apart; using __tbss_size alone made adjacent harts' images overlap by - __tbss_offset bytes. __tbss_offset and __tbss_size are both - section-aligned, so their sum keeps the image alignment. */ + __tbss_offset bytes. .tdata is aligned like .tbss: when it is empty + the linker drops it and the TLS image starts at .tbss, so an unaligned + __tdata_start would skew __tbss_offset and the stride off the 8-byte + boundary that every hart's tp must keep. */ PROVIDE(__tls_block_size = __tbss_offset + __tbss_size); .preinit_array : { From 97a9e19b15e9a96e6051ca400e3a89292e98bbb6 Mon Sep 17 00:00:00 2001 From: tinebp Date: Sun, 25 Oct 2026 21:10:03 -0700 Subject: [PATCH 6/6] simx: forward the amocas comparand through the local-memory switch LocalMemSwitch rebuilt each lane's request for the dcache and local-memory paths without copying amo_cmp, so amocas always compared against 0: a CAS with a nonzero expected value never matched, and CAS-increment loops stalled at 1. RTL was correct. The amo regression cases only use an expected value of 0, which is why they passed. amocas(expected 9 -> 5) probe: SimX now matches rtlsim. Co-Authored-By: Claude Opus 5.5 --- sim/simx/mem/local_mem_switch.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index 7cc253575e..0edd322f21 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -88,11 +88,13 @@ void LocalMemSwitch::on_tick() { out_lmem_req.addrs.at(i) = in_req.addrs.at(i); out_lmem_req.data.at(i) = in_req.data.at(i); out_lmem_req.byteen.at(i) = in_req.byteen.at(i); + out_lmem_req.amo_cmp.at(i) = in_req.amo_cmp.at(i); } else { out_dc_req.mask.set(i); out_dc_req.addrs.at(i) = in_req.addrs.at(i); out_dc_req.data.at(i) = in_req.data.at(i); out_dc_req.byteen.at(i) = in_req.byteen.at(i); + out_dc_req.amo_cmp.at(i) = in_req.amo_cmp.at(i); } } }