From 35abdfa0b2d9fb75beb04a0154e79c21b0d296d8 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 00:14:16 -0700 Subject: [PATCH 1/9] mufu: rcpf, rsqrt, ex2, lg2, sin, cos, tanh --- VX_config.toml | 10 +- VX_types.toml | 8 + ci/baselines/synthesis/xilinx/mufu.json | 174 ++++++ ci/testcases/fpga_gate.yaml | 10 + ci/testcases/mufu.yaml | 59 ++ ci/testcases/unittest.yaml | 6 + .../custom_accelerator_isa_extensions.md | 21 + docs/designs/floating_point_unit.md | 8 + hw/rtl/VX_define.vh | 6 + hw/rtl/VX_gpu_pkg.sv | 29 +- hw/rtl/VX_trace_pkg.sv | 22 + hw/rtl/core/VX_csr_data.sv | 9 + hw/rtl/core/VX_decode.sv | 12 + hw/rtl/core/VX_execute.sv | 11 + hw/rtl/core/VX_lane_dispatch.sv | 5 +- hw/rtl/libs/VX_nz_iterator.sv | 5 +- hw/rtl/mufu/VX_mufu_pe.sv | 373 ++++++++++++ hw/rtl/mufu/VX_mufu_rom.sv | 559 ++++++++++++++++++ hw/rtl/mufu/VX_mufu_unit.sv | 127 ++++ hw/scripts/gen_mufu_tables.py | 311 ++++++++++ hw/syn/extensions.mk | 5 + hw/syn/xilinx/dut/catalog.mk | 9 +- hw/unittest/Makefile | 7 + hw/unittest/mufu_unit/Makefile | 26 + hw/unittest/mufu_unit/VX_mufu_unit_top.sv | 37 ++ hw/unittest/mufu_unit/main.cpp | 112 ++++ sim/avedsim/Makefile | 5 + sim/opaesim/Makefile | 5 + sim/rtlsim/Makefile | 5 + sim/simx/Makefile | 6 + sim/simx/core.cpp | 23 +- sim/simx/core.h | 6 + sim/simx/csr_unit.cpp | 8 + sim/simx/decode.cpp | 18 + sim/simx/mufu_unit.cpp | 74 +++ sim/simx/mufu_unit.h | 36 ++ sim/simx/types.h | 46 ++ sim/xrtsim/Makefile | 5 + sw/common/mufu.cpp | 204 +++++++ sw/common/mufu.h | 38 ++ sw/common/mufu_tables.h | 538 +++++++++++++++++ sw/kernel/include/vx_mufu.h | 106 ++++ sw/runtime/common/perf.cpp | 18 + sw/runtime/include/vortex2.h | 1 + tests/regression/Makefile | 2 +- tests/regression/mufu/Makefile | 20 + tests/regression/mufu/common.h | 36 ++ tests/regression/mufu/kernel.cpp | 41 ++ tests/regression/mufu/main.cpp | 302 ++++++++++ 49 files changed, 3496 insertions(+), 8 deletions(-) create mode 100644 ci/baselines/synthesis/xilinx/mufu.json create mode 100644 ci/testcases/mufu.yaml create mode 100644 hw/rtl/mufu/VX_mufu_pe.sv create mode 100644 hw/rtl/mufu/VX_mufu_rom.sv create mode 100644 hw/rtl/mufu/VX_mufu_unit.sv create mode 100644 hw/scripts/gen_mufu_tables.py create mode 100644 hw/unittest/mufu_unit/Makefile create mode 100644 hw/unittest/mufu_unit/VX_mufu_unit_top.sv create mode 100644 hw/unittest/mufu_unit/main.cpp create mode 100644 sim/simx/mufu_unit.cpp create mode 100644 sim/simx/mufu_unit.h create mode 100644 sw/common/mufu.cpp create mode 100644 sw/common/mufu.h create mode 100644 sw/common/mufu_tables.h create mode 100644 sw/kernel/include/vx_mufu.h create mode 100644 tests/regression/mufu/Makefile create mode 100644 tests/regression/mufu/common.h create mode 100644 tests/regression/mufu/kernel.cpp create mode 100644 tests/regression/mufu/main.cpp diff --git a/VX_config.toml b/VX_config.toml index f51186cdec..f999fad140 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -41,6 +41,7 @@ VX_CFG_EXT_TEX_ENABLE = false VX_CFG_EXT_RASTER_ENABLE = false VX_CFG_EXT_OM_ENABLE = false VX_CFG_EXT_RTU_ENABLE = false +VX_CFG_EXT_MUFU_ENABLE = false VX_CFG_VLEN = "expr: $VX_CFG_XLEN * 4" [pipeline] @@ -149,6 +150,13 @@ VX_CFG_FSQRT_PE_RATIO= 1 VX_CFG_FCVT_PE_RATIO = 1 VX_CFG_FNCP_PE_RATIO = 1 +[mufu] +# Special-function unit (rcp/rsqrt/ex2/lg2/sinpi/cospi/tanh): one lane shared by +# all issue slots; a wider unit did not speed up fused FlashAttention. +VX_CFG_NUM_MUFU_LANES = 1 +VX_CFG_NUM_MUFU_BLOCKS = 1 +VX_CFG_MUFU_LATENCY = 7 + [amo] # LR/SC reservation stations per LLC bank; line-indexed in BRAM, eviction yields legal spurious SC failures VX_CFG_AMO_RS_SIZE = "expr: max($VX_CFG_NUM_WARPS, up($VX_CFG_NUM_CORES * $VX_CFG_NUM_WARPS / $VX_CFG_L2_NUM_BANKS))" @@ -447,7 +455,7 @@ VX_CFG_NUM_OCACHES = "expr: max(1, up($VX_CFG_NUM_OM_CORES / 4))" VX_CFG_MISA_STD = "expr: ($VX_CFG_EXT_A_ENABLED << 0) | (0 << 1) | ($VX_CFG_EXT_C_ENABLED << 2) | ($VX_CFG_EXT_D_ENABLED << 3) | (0 << 4) | ($VX_CFG_EXT_F_ENABLED << 5) | (0 << 6) | (0 << 7) | (1 << 8) | (0 << 9) | (0 << 10) | (0 << 11) | ($VX_CFG_EXT_M_ENABLED << 12) | (0 << 13) | (0 << 14) | (0 << 15) | (0 << 16) | (0 << 17) | (0 << 18) | (0 << 19) | (1 << 20) | (0 << 21) | (0 << 22) | (1 << 23) | (0 << 24) | (0 << 25)" # Custom extensions signature -VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11)" +VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11) | ($VX_CFG_EXT_MUFU_ENABLED << 12)" [toolchain] ASIC = false diff --git a/VX_types.toml b/VX_types.toml index 7143523976..9ab6e5f2c6 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -592,6 +592,7 @@ VX_DCR_MPM_CLASS_TEX = 13 VX_DCR_MPM_CLASS_OM = 14 VX_DCR_MPM_CLASS_RTU = 15 VX_DCR_MPM_CLASS_DXA = 16 +VX_DCR_MPM_CLASS_MUFU = 17 [csr_mpm_base] VX_CSR_MCYCLE = 0xB00 @@ -785,6 +786,13 @@ VX_CSR_MPM_TCU_TBUF_CACHE_HITS_H = 0xB84 VX_CSR_MPM_TCU_LMEM_READS = 0xB05 # tile buffer local memory reads VX_CSR_MPM_TCU_LMEM_READS_H = 0xB85 +[csr_mpm_mufu] +# PERF: special-function unit (per-core; summed across issue slots) +VX_CSR_MPM_MUFU_STALLS = 0xB03 # dispatch stalled due to MUFU pipeline full +VX_CSR_MPM_MUFU_STALLS_H = 0xB83 +VX_CSR_MPM_MUFU_INSTRS = 0xB04 # total MUFU instructions +VX_CSR_MPM_MUFU_INSTRS_H = 0xB84 + [csr_mpm_tex] # PERF: texture unit (per-core; summed across blocks) VX_CSR_MPM_TEX_READS = 0xB03 # texture accesses diff --git a/ci/baselines/synthesis/xilinx/mufu.json b/ci/baselines/synthesis/xilinx/mufu.json new file mode 100644 index 0000000000..1faf73164c --- /dev/null +++ b/ci/baselines/synthesis/xilinx/mufu.json @@ -0,0 +1,174 @@ +{ + "mufu": { + "clock_mhz": 300, + "config_hash": "5d5eb85429e40b51", + "configs": "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16", + "dut": "mufu", + "result": { + "bram": 1, + "build_time_s": 885, + "critical_paths": [ + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__33/D", + "group": "core_clock", + "levels": 8, + "logic_ns": 0.924, + "route_ns": 1.721, + "route_pct": 65.069, + "slack_ns": 0.668, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__33/D", + "group": "core_clock", + "levels": 11, + "logic_ns": 0.984, + "route_ns": 1.652, + "route_pct": 62.675, + "slack_ns": 0.677, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__34/D", + "group": "core_clock", + "levels": 8, + "logic_ns": 0.9, + "route_ns": 1.723, + "route_pct": 65.691, + "slack_ns": 0.69, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__32/D", + "group": "core_clock", + "levels": 8, + "logic_ns": 0.894, + "route_ns": 1.721, + "route_pct": 65.815, + "slack_ns": 0.698, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__34/D", + "group": "core_clock", + "levels": 11, + "logic_ns": 0.96, + "route_ns": 1.654, + "route_pct": 63.279, + "slack_ns": 0.699, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__31/D", + "group": "core_clock", + "levels": 8, + "logic_ns": 0.89, + "route_ns": 1.722, + "route_pct": 65.929, + "slack_ns": 0.701, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__32/D", + "group": "core_clock", + "levels": 11, + "logic_ns": 0.954, + "route_ns": 1.652, + "route_pct": 63.396, + "slack_ns": 0.707, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__29/D", + "group": "core_clock", + "levels": 8, + "logic_ns": 0.884, + "route_ns": 1.721, + "route_pct": 66.068, + "slack_ns": 0.708, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__42/D", + "group": "core_clock", + "levels": 7, + "logic_ns": 0.909, + "route_ns": 1.696, + "route_pct": 65.108, + "slack_ns": 0.708, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + }, + { + "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__40/D", + "group": "core_clock", + "levels": 7, + "logic_ns": 0.909, + "route_ns": 1.695, + "route_pct": 65.095, + "slack_ns": 0.709, + "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + } + ], + "dsp": 3, + "env": { + "device": "xcu55c-fsvh2892-2L-e", + "opt_level": 3, + "vivado": "2024.2", + "xlen": 32 + }, + "ff": 1153, + "fmax_mhz": 375.2, + "high_fanout_nets": [ + { + "driver": "LUT3", + "fanout": 608, + "net": "mufu_unit/g_blocks[0].pe_serializer/out_buf/g_eb1.pipe_buffer/g_register.g_pipe_regs[0].pipe_register/g_pipe.g_partial_reset.pipe_rst_reg[0][0]_1" + }, + { + "driver": "LUT2", + "fanout": 233, + "net": "mufu_unit/g_blocks[0].pe_serializer/out_buf/g_eb1.pipe_buffer/g_register.g_pipe_regs[0].pipe_register/g_buffer.flow_out" + }, + { + "driver": "LUT5", + "fanout": 149, + "net": "mufu_unit/lane_dispatch/g_blocks[0].g_partial_simd.packet_iter/g_iterator.pipe_reg/g_iterator.sent_mask_p_reg[7][1]" + }, + { + "driver": "LUT5", + "fanout": 149, + "net": "mufu_unit/lane_dispatch/g_blocks[0].g_partial_simd.packet_iter/g_iterator.pipe_reg/g_iterator.sent_mask_p_reg[7][0]" + }, + { + "driver": "FDRE", + "fanout": 111, + "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/per_block_execute_if[0]\\\\.data[op_type][1]" + }, + { + "driver": "LUT3", + "fanout": 108, + "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r[88]_i_1_n_0" + }, + { + "driver": "FDRE", + "fanout": 107, + "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/per_block_execute_if[0]\\\\.data[op_type][0]" + }, + { + "driver": "FDSE", + "fanout": 106, + "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_0" + }, + { + "driver": "LUT3", + "fanout": 104, + "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r[87]_i_1_n_0" + } + ], + "lut": 1227, + "lutram": 57, + "uram": 0, + "wns_ns": 0.668 + } + } +} diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index aba38f15d9..0905d5503a 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -103,6 +103,16 @@ builds: clock_mhz: 300 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# --- mufu -------------------------------------------------------------------- + +# Special-function unit at its default single lane under a 16-wide warp: lane +# dispatch, the 7-stage table+polynomial datapath and the 512x72 table ROM. +- id: mufu + group: mufu + dut: mufu + clock_mhz: 300 + configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16" + # --- tensor ------------------------------------------------------------------ # TFR tensor core with every datatype plus sparsity and WGMMA. Widest FEDP/DSP48 diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml new file mode 100644 index 0000000000..c17a5d5589 --- /dev/null +++ b/ci/testcases/mufu.yaml @@ -0,0 +1,59 @@ +# MUFU special-function unit (custom-2: rcp/rsqrt/ex2/lg2/sinpi/cospi/tanh). +# The `mufu` app self-enables VX_CFG_EXT_MUFU_ENABLE and checks every result +# bit-exactly against the shared golden model (sw/common/mufu.cpp) plus libm +# accuracy bounds; the datapath itself is covered by unittest:hw-mufu. +category: mufu +defaults: + xlen: + - 32 + - 64 + tier: smoke + touches: [hw/rtl/mufu, sim/simx/mufu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] +tests: +- id: mufu-1 + via: blackbox + drivers: + - simx + app: mufu +# Wide warp: 16 threads over 1 MUFU lane, so every warp serializes into 16 packets. +- id: mufu-nt16 + via: blackbox + drivers: + - simx + - rtlsim + app: mufu + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# Full-rate lanes (non-default): no lane-gather stage on the response path. +- id: mufu-full-lanes + via: blackbox + drivers: + - simx + - rtlsim + app: mufu + configs: -DVX_CFG_NUM_MUFU_LANES=4 +- id: mufu-xrt + via: blackbox + drivers: + - xrt + app: mufu + +# --------------------------------------------------------------------------- +# check: model_parity — SimX vs rtlsim (see core.yaml). Each run is sized to +# >=~300k cycles so steady-state MUFU throughput dominates. +# --------------------------------------------------------------------------- + +# Default shape: 4 threads over the 1 lane, chained ex2 (~609k cycles). +- id: parity-mufu + check: model_parity + via: blackbox + app: mufu + args: -n1024 -r256 -tex2 +# 8 threads over 1 lane (8 packets per warp), chained sinpi (~670k cycles). +# Not 16 threads: there SimX runs ~5% fast on saturated multi-packet MUFU +# issue. +- id: parity-mufu-nt8 + check: model_parity + via: blackbox + app: mufu + args: -n2048 -r256 -tsinpi + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 89c8ba08e8..be00b76514 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -38,3 +38,9 @@ tests: via: script run: "make -C hw/unittest/tex_lerp run" touches: [hw/rtl/tex, hw/unittest/tex_lerp, sw/common/vx_gfx_abi.h, sw/common/gfx_frag_tex.h] + # MUFU datapath (VX_mufu_pe) against the golden model SimX and the tests share + # (sw/common/mufu.cpp): specials, a strided sweep of all FP32 inputs, random. + - id: hw-mufu + via: script + run: "make -C hw/unittest/mufu_unit run" + touches: [hw/rtl/mufu, hw/unittest/mufu_unit, sw/common/mufu.cpp, sw/common/mufu_tables.h] diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index 7461d1677d..a4593709a3 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -214,6 +214,27 @@ A register **field** may also carry an immediate. The tensor instructions put formats and flags in `rd`, `rs1` and `rs2`; decode reads the field without reading a register. +### Opcode map + +Allocated custom opcode space. Check here before claiming a slot, and update +the table in the same change. + +| Opcode | Split by | Value | Extension | Format | +|---|---|---|---|---| +| custom-0 `0x0B` | `funct7` | `0x00` | warp control (`tmc`, `wspawn`, `split`, `join`, `bar`, `pred`, `wsync`) | R | +| | | `0x01` | vote / shuffle | R | +| | | `0x02` | tensor core (WMMA/WGMMA) | R | +| | | `0x03` | DXA async copy | R | +| | | `0x04` | packed loads (`vx_packl*`) | R | +| custom-1 `0x2B` | `funct3` | `0` | `vx_wgather` | R4 | +| | | `3` | `vx_om_export` | R4 | +| | | `5` | `vx_tex` | R4 | +| | | `6` | RTU hit window / callback return | R4 | +| | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | +| | | `1`, `2`, `4` | free | | +| custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | +| custom-3 `0x7B` | | | free | | + --- ## 4. How a wide-operand instruction executes diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index 65cd877c8a..56fd16c78d 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -444,6 +444,14 @@ divergence from soft-float fails CI. exercise the FPU's arithmetic RTL. `isa-32f-dsp` is the one lane that does, and it is F32 only: the F64 soft cores are covered by the unit tests alone. +### Special functions are not in the FPU + +Approximate transcendentals (`rcp`, `rsqrt`, `ex2`, `lg2`, `sinpi`, `cospi`, +`tanh`) run on a separate execution unit, the MUFU, not on an FPU core. It has +its own dispatch queue and commit path, so FPU and MUFU instructions from +different warps execute concurrently; it ignores `frm` and never writes +`fflags`. The FPU's `fsqrt`/`fdiv` remain the correctly rounded operations. + --- ## 13. Known limitations diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index b2bf9f07c2..0dfd4dffe1 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -95,6 +95,12 @@ `endif `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + `ifndef VX_CFG_EXT_F_ENABLE + `error "VX_CFG_EXT_MUFU_ENABLE requires VX_CFG_EXT_F_ENABLE" + `endif +`endif + // Convenience flag: the TCU metadata SRAM is present when any metadata-consuming // mode (MX or sparse) is enabled. Internal derived macro — not a VX_CFG_* knob. `ifdef VX_CFG_TCU_MX_ENABLE diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index d836c5a760..73a49bf371 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -240,8 +240,9 @@ package VX_gpu_pkg; localparam EX_SFU = 2; localparam EX_FPU = (EX_SFU + `VX_CFG_EXT_F_ENABLED); localparam EX_TCU = (EX_FPU + `VX_CFG_EXT_TCU_ENABLED); + localparam EX_MUFU = (EX_TCU + `VX_CFG_EXT_MUFU_ENABLED); - localparam NUM_EX_UNITS = EX_TCU + 1; + localparam NUM_EX_UNITS = EX_MUFU + 1; localparam EX_BITS = `CLOG2(NUM_EX_UNITS); localparam EX_WIDTH = `UP(EX_BITS); @@ -546,6 +547,18 @@ package VX_gpu_pkg; return (op >= 6 && op <= 8); endfunction +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam INST_MUFU_RCP = 4'h0; + localparam INST_MUFU_RSQRT = 4'h1; + localparam INST_MUFU_EX2 = 4'h2; + localparam INST_MUFU_LG2 = 4'h3; + localparam INST_MUFU_SINPI = 4'h4; + localparam INST_MUFU_COSPI = 4'h5; + localparam INST_MUFU_TANH = 4'h6; + localparam INST_MUFU_BITS = 4; + localparam INST_MUFU_FMT_BITS = 3; +`endif + /////////////////////////////// Issue parameters ////////////////////////// localparam ISSUE_ISW_BITS = `CLOG2(`VX_CFG_ISSUE_WIDTH); @@ -959,6 +972,14 @@ package VX_gpu_pkg; `PACKAGE_ASSERT($bits(tcu_args_t) == INST_ARGS_BITS) `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + typedef struct packed { + logic [INST_ARGS_BITS-INST_MUFU_FMT_BITS-1:0] __padding; + logic [INST_MUFU_FMT_BITS-1:0] fmt; + } mufu_args_t; + `PACKAGE_ASSERT($bits(mufu_args_t) == INST_ARGS_BITS) +`endif + `ifdef VX_CFG_EXT_TEX_ENABLE // Texture sample op args (vx_tex, CUSTOM1 funct3=5, R4-type): u/v/lod ride // rs1/rs2/rs3, the texel lands in rd, and `stage` (funct2) selects the @@ -1018,6 +1039,9 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_TCU_ENABLE tcu_args_t tcu; `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + mufu_args_t mufu; + `endif `ifdef VX_CFG_EXT_TEX_ENABLE tex_args_t tex; `endif @@ -1194,6 +1218,9 @@ package VX_gpu_pkg; `DECL_EXECUTE_T (alu, `VX_CFG_NUM_ALU_LANES); `DECL_EXECUTE_T (lsu, `VX_CFG_NUM_LSU_LANES); `DECL_EXECUTE_T (sfu, `VX_CFG_NUM_SFU_LANES); +`ifdef VX_CFG_EXT_MUFU_ENABLE + `DECL_EXECUTE_T (mufu, `VX_CFG_NUM_MUFU_LANES); +`endif //////////////////////////// Perf counter types /////////////////////////// diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index dd16cd3dc7..39fb310cec 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -36,6 +36,9 @@ package VX_trace_pkg; `endif `ifdef VX_CFG_EXT_TCU_ENABLE EX_TCU: `TRACE(level, ("TCU")) + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + EX_MUFU: `TRACE(level, ("MUFU")) `endif default: `TRACE(level, ("?")) endcase @@ -431,6 +434,20 @@ package VX_trace_pkg; EX_TCU: begin VX_tcu_pkg::trace_ex_op(level, op_type, op_args); end + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + EX_MUFU: begin + case (INST_MUFU_BITS'(op_type)) + INST_MUFU_RCP: `TRACE(level, ("MUFU.RCP")) + INST_MUFU_RSQRT: `TRACE(level, ("MUFU.RSQRT")) + INST_MUFU_EX2: `TRACE(level, ("MUFU.EX2")) + INST_MUFU_LG2: `TRACE(level, ("MUFU.LG2")) + INST_MUFU_SINPI: `TRACE(level, ("MUFU.SINPI")) + INST_MUFU_COSPI: `TRACE(level, ("MUFU.COSPI")) + INST_MUFU_TANH: `TRACE(level, ("MUFU.TANH")) + default: `TRACE(level, ("?")) + endcase + end `endif default: `TRACE(level, ("?")) endcase @@ -457,6 +474,11 @@ package VX_trace_pkg; EX_FPU: begin `TRACE(level, (", fmt=0x%0h, frm=0x%0h", op_args.fpu.fmt, op_args.fpu.frm)) end + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + EX_MUFU: begin + `TRACE(level, (", fmt=0x%0h", op_args.mufu.fmt)) + end `endif default:; endcase diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index f09c17bb7e..668c5f7ac0 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -529,6 +529,15 @@ import VX_fpu_pkg::*; endcase end `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + `VX_DCR_MPM_CLASS_MUFU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_MUFU_STALLS, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_MUFU]); + `CSR_READ_64(`VX_CSR_MPM_MUFU_INSTRS, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_MUFU]); + default:; + endcase + end + `endif `ifdef VX_CFG_EXT_TEX_ENABLE `VX_DCR_MPM_CLASS_TEX: begin case (dcr_addr) diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 9c2a3fa44d..25cbd3dcdd 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -880,6 +880,18 @@ module VX_decode import default:; endcase end + `ifdef VX_CFG_EXT_MUFU_ENABLE + INST_EXT3: begin + if (funct7 == 7'h00 && funct3 == 3'h0 && rs2 <= 5'(INST_MUFU_TANH)) begin + ex_type = EX_MUFU; + op_type = INST_OP_BITS'(rs2); + op_args.mufu.__padding = '0; + op_args.mufu.fmt = funct3; + `USED_FREG (rd); + `USED_FREG (rs1); + end + end + `endif default:; endcase end diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 4f0222e066..f7eb8fec39 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -142,6 +142,17 @@ module VX_execute import VX_gpu_pkg::*; #( ); `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + VX_mufu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))) + ) mufu_unit ( + .clk (clk), + .reset (reset), + .dispatch_if (dispatch_if[EX_MUFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), + .commit_if (commit_if[EX_MUFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) + ); +`endif + VX_sfu_unit #( .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))), .CORE_ID (CORE_ID) diff --git a/hw/rtl/core/VX_lane_dispatch.sv b/hw/rtl/core/VX_lane_dispatch.sv index b6ae6126a1..664df578b5 100644 --- a/hw/rtl/core/VX_lane_dispatch.sv +++ b/hw/rtl/core/VX_lane_dispatch.sv @@ -64,6 +64,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign dispatch_if[i].ready = dispatch_ready[i]; end + wire [BLOCK_SIZE-1:0] block_valid; wire [BLOCK_SIZE-1:0] block_ready; wire [BLOCK_SIZE-1:0][NUM_LANES-1:0] block_tmask; wire [BLOCK_SIZE-1:0][2:0][NUM_LANES-1:0][`VX_CFG_XLEN-1:0] block_rsdata; @@ -177,6 +178,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign block_pid[block_idx] = start_p; assign block_sop[block_idx] = is_first_p; assign block_eop[block_idx] = is_last_p; + assign block_valid[block_idx] = valid_p; assign block_ready[block_idx] = ready_p; assign block_done[block_idx] = (fire_p && is_last_p) || ~dispatch_valid[issue_idx]; end else begin : g_full_simd @@ -186,6 +188,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign block_pid[block_idx] = 0; assign block_sop[block_idx] = 1; assign block_eop[block_idx] = 1; + assign block_valid[block_idx] = valid_p; assign block_ready[block_idx] = ready_p; assign block_done[block_idx] = ready_p || ~valid_p; end @@ -240,7 +243,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( always @(*) begin ready_in = 0; for (integer block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin - ready_in[issue_indices[block_idx]] = block_ready[block_idx] && block_eop[block_idx]; + ready_in[issue_indices[block_idx]] = block_valid[block_idx] && block_ready[block_idx] && block_eop[block_idx]; end end assign dispatch_ready = ready_in; diff --git a/hw/rtl/libs/VX_nz_iterator.sv b/hw/rtl/libs/VX_nz_iterator.sv index 5a364b7166..f0348a2c34 100644 --- a/hw/rtl/libs/VX_nz_iterator.sv +++ b/hw/rtl/libs/VX_nz_iterator.sv @@ -73,9 +73,10 @@ module VX_nz_iterator #( wire is_last_p = (start_p == end_p); wire enable = valid_in && (~valid_out || next); + wire eop_out = valid_out && eop; always @(posedge clk) begin - if (reset || (enable && (is_last_p || eop))) begin + if (reset || (enable && (is_last_p || eop_out))) begin sent_mask_p <= '0; is_first_p <= 1; end else if (enable) begin @@ -90,7 +91,7 @@ module VX_nz_iterator #( .DEPTH (OUT_REG) ) pipe_reg ( .clk (clk), - .reset (reset || (enable && eop)), + .reset (reset || (enable && eop_out)), .enable (enable), .data_in ({valid_in, data_in[start_p], start_p, is_first_p, is_last_p}), .data_out ({valid_out, data_out, pid, sop, eop}) diff --git a/hw/rtl/mufu/VX_mufu_pe.sv b/hw/rtl/mufu/VX_mufu_pe.sv new file mode 100644 index 0000000000..24619244be --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_pe.sv @@ -0,0 +1,373 @@ +`include "VX_define.vh" + +module VX_mufu_pe import VX_gpu_pkg::*; #( + parameter LATENCY = 7 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire [2:0] fn, + input wire [31:0] dataa, + output wire [31:0] result +); + `STATIC_ASSERT (LATENCY == 7, ("VX_mufu_pe: LATENCY must be 7, got %0d", LATENCY)) + + localparam TBL_RCP = 3'd0; + localparam TBL_RSQRT = 3'd1; + localparam TBL_EX2 = 3'd2; + localparam TBL_LG2 = 3'd3; + localparam TBL_SIN = 3'd4; + localparam TBL_TANH = 3'd5; + + localparam F32_QNAN = 32'h7fc00000; + localparam F32_POS_INF = 32'h7f800000; + localparam F32_NEG_INF = 32'hff800000; + localparam F32_ONE = 32'h3f800000; + + wire x_s = dataa[31]; + wire [7:0] x_e = dataa[30:23]; + wire [22:0] x_m = dataa[22:0]; + wire [23:0] x_sig = {1'b1, x_m}; + + wire x_zero = (x_e == 8'd0); + wire x_inf = (x_e == 8'hff) && (x_m == 23'd0); + wire x_nan = (x_e == 8'hff) && (x_m != 23'd0); + + wire signed [9:0] x_k = 10'($signed({2'b00, x_e})) - 10'sd127; + + wire [3:0] ex2_lsh = 4'(x_e - 8'd120); + wire [7:0] ex2_rsh = 8'd120 - x_e; + wire [37:0] ex2_mag = (x_e >= 8'd120) ? (38'(x_sig) << ex2_lsh) + : ((ex2_rsh < 8'd24) ? 38'(x_sig >> ex2_rsh[4:0]) : 38'd0); + wire [38:0] ex2_x = x_s ? -{1'b0, ex2_mag} : {1'b0, ex2_mag}; + + wire [7:0] trig_lsh = x_e - 8'd119; + wire [7:0] trig_rsh = 8'd119 - x_e; + wire is_sin = (fn == 3'(INST_MUFU_SINPI)); + wire [31:0] trig_u0 = (x_e >= 8'd119) ? ((trig_lsh < 8'd32) ? (32'(x_sig) << trig_lsh[4:0]) : 32'd0) + : ((trig_rsh < 8'd24) ? 32'(x_sig >> trig_rsh[4:0]) : 32'd0); + wire [31:0] trig_u = is_sin ? trig_u0 : (trig_u0 + 32'h40000000); + wire trig_odd = trig_u[30]; + wire [29:0] trig_t = trig_odd ? ~trig_u[29:0] : trig_u[29:0]; + wire trig_neg = trig_u[31] ^ (is_sin && x_s); + wire trig_one = trig_odd && (trig_u[29:0] == 30'd0); + `UNUSED_VAR ({ex2_x[5:0], trig_t[5:0]}) + + reg r_special; + reg [31:0] r_sval; + reg [2:0] r_tbl; + reg [6:0] r_idx; + reg [17:0] r_xl; + reg signed [9:0] r_exp; + reg r_neg; + reg r_zsign; + reg r_lg2; + + always @(*) begin + r_special = 1'b0; + r_sval = 32'd0; + r_tbl = TBL_RCP; + r_idx = x_m[22:16]; + r_xl = {x_m[15:0], 2'b00}; + r_exp = 10'sd0; + r_neg = 1'b0; + r_zsign = 1'b0; + r_lg2 = 1'b0; + case (fn) + 3'(INST_MUFU_RCP): begin + r_neg = x_s; + r_exp = 10'sd127 - 10'($signed({2'b00, x_e})); + if (x_inf) begin + r_special = 1'b1; + r_sval = {x_s, 31'd0}; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = x_s ? F32_NEG_INF : F32_POS_INF; + end + end + 3'(INST_MUFU_RSQRT): begin + r_tbl = TBL_RSQRT; + r_idx = {x_k[0], x_m[22:17]}; + r_xl = {x_m[16:0], 1'b0}; + r_exp = -(x_k >>> 1); + if (x_zero) begin + r_special = 1'b1; + r_sval = x_s ? F32_NEG_INF : F32_POS_INF; + end else if (x_s) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_inf) begin + r_special = 1'b1; + r_sval = 32'd0; + end + end + 3'(INST_MUFU_EX2): begin + r_tbl = TBL_EX2; + r_idx = {1'b0, ex2_x[29:24]}; + r_xl = ex2_x[23:6]; + r_exp = 10'($signed(ex2_x[38:30])); + if (x_inf) begin + r_special = 1'b1; + r_sval = x_s ? 32'd0 : F32_POS_INF; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = F32_ONE; + end else if (x_e >= 8'd134) begin + r_special = 1'b1; + r_sval = x_s ? 32'd0 : F32_POS_INF; + end + end + 3'(INST_MUFU_LG2): begin + r_tbl = TBL_LG2; + r_idx = {1'b0, x_m[22:17]}; + r_xl = {x_m[16:0], 1'b0}; + r_lg2 = 1'b1; + if (x_zero) begin + r_special = 1'b1; + r_sval = F32_NEG_INF; + end else if (x_s) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_inf) begin + r_special = 1'b1; + r_sval = F32_POS_INF; + end + end + 3'(INST_MUFU_SINPI), + 3'(INST_MUFU_COSPI): begin + r_tbl = TBL_SIN; + r_idx = {1'b0, trig_t[29:24]}; + r_xl = trig_t[23:6]; + r_neg = trig_neg; + r_zsign = is_sin && x_s; + if (x_inf) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = is_sin ? {x_s, 31'd0} : F32_ONE; + end else if (trig_one) begin + r_special = 1'b1; + r_sval = {trig_neg, F32_ONE[30:0]}; + end + end + 3'(INST_MUFU_TANH): begin + r_tbl = TBL_TANH; + r_idx = {1'b0, 3'(x_e - 8'd122), x_m[22:20]}; + r_xl = x_m[19:2]; + r_exp = x_k + 10'sd4; + r_neg = x_s; + if (x_e >= 8'd130) begin + r_special = 1'b1; + r_sval = {x_s, F32_ONE[30:0]}; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = {x_s, 31'd0}; + end else if (x_e < 8'd122) begin + r_special = 1'b1; + r_sval = dataa; + end + end + default: begin + r_special = 1'b1; + r_sval = 32'd0; + end + endcase + if (x_nan) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end + end + + localparam MISC_W = 1 + 32 + 10 + 1 + 1 + 1 + 10; + + wire [MISC_W-1:0] s1_misc, s2_misc, s3_misc, s4_misc; + wire [2:0] s1_tbl; + wire [6:0] s1_idx; + wire [17:0] s1_xl, s2_xl, s3_xl; + + VX_pipe_register #( + .DATAW (MISC_W + 3 + 7 + 18) + ) s1_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({r_special, r_sval, r_exp, r_neg, r_zsign, r_lg2, x_k, r_tbl, r_idx, r_xl}), + .data_out ({s1_misc, s1_tbl, s1_idx, s1_xl}) + ); + + wire [31:0] rom_a0; + wire signed [26:0] rom_a1; + wire signed [19:0] rom_a2; + + VX_mufu_rom rom ( + .clk (clk), + .enable (enable), + .tbl (s1_tbl), + .idx (s1_idx), + .a0 (rom_a0), + .a1 (rom_a1), + .a2 (rom_a2) + ); + + VX_pipe_register #( + .DATAW (MISC_W + 18) + ) s2_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s1_misc, s1_xl}), + .data_out ({s2_misc, s2_xl}) + ); + + wire [35:0] xl_sq = 36'(s2_xl) * 36'(s2_xl); + `UNUSED_VAR (xl_sq[17:0]) + + wire [17:0] s3_sq; + + VX_pipe_register #( + .DATAW (MISC_W + 18 + 18) + ) s3_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s2_misc, s2_xl, xl_sq[35:18]}), + .data_out ({s3_misc, s3_xl, s3_sq}) + ); + + wire signed [45:0] s4_p1; + wire signed [38:0] s4_p2; + + VX_multiplier #( + .A_WIDTH (27), + .B_WIDTH (19), + .R_WIDTH (46), + .SIGNED (1), + .LATENCY (1) + ) mul1 ( + .clk (clk), + .enable (enable), + .dataa (rom_a1), + .datab ({1'b0, s3_xl}), + .result (s4_p1) + ); + + VX_multiplier #( + .A_WIDTH (20), + .B_WIDTH (19), + .R_WIDTH (39), + .SIGNED (1), + .LATENCY (1) + ) mul2 ( + .clk (clk), + .enable (enable), + .dataa (rom_a2), + .datab ({1'b0, s3_sq}), + .result (s4_p2) + ); + + wire [31:0] s4_a0; + + VX_pipe_register #( + .DATAW (MISC_W + 32) + ) s4_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s3_misc, rom_a0}), + .data_out ({s4_misc, s4_a0}) + ); + + wire s4_special, s4_neg, s4_zsign, s4_lg2; + wire [31:0] s4_sval; + wire signed [9:0] s4_exp, s4_k; + assign {s4_special, s4_sval, s4_exp, s4_neg, s4_zsign, s4_lg2, s4_k} = s4_misc; + + wire signed [27:0] t1 = 28'(s4_p1 >>> 18); + wire signed [20:0] t2 = 21'(s4_p2 >>> 18); + wire signed [34:0] p_sum = $signed({3'b000, s4_a0}) + 35'(t1) + 35'(t2); + wire [33:0] p_pos = p_sum[34] ? 34'd0 : p_sum[33:0]; + wire signed [40:0] lg2_sum = 41'($signed({s4_k, 30'd0})) + $signed({7'd0, p_pos}); + wire lg2_neg = lg2_sum[40]; + wire [39:0] lg2_mag = 40'(lg2_neg ? -lg2_sum : lg2_sum); + + wire [39:0] mag = s4_lg2 ? lg2_mag : 40'(p_pos); + wire neg = s4_lg2 ? lg2_neg : s4_neg; + wire signed [9:0] exp_adj = s4_lg2 ? 10'sd0 : s4_exp; + wire zsign = s4_lg2 ? 1'b0 : s4_zsign; + + wire s5_special, s5_neg, s5_zsign; + wire [31:0] s5_sval; + wire signed [9:0] s5_exp; + wire [39:0] s5_mag; + + VX_pipe_register #( + .DATAW (1 + 32 + 10 + 1 + 1 + 40) + ) s5_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s4_special, s4_sval, exp_adj, neg, zsign, mag}), + .data_out ({s5_special, s5_sval, s5_exp, s5_neg, s5_zsign, s5_mag}) + ); + + wire [5:0] lz; + wire mag_nz; + + VX_lzc #( + .N (40) + ) lzc ( + .data_in (s5_mag), + .data_out (lz), + .valid_out (mag_nz) + ); + + wire [39:0] norm = s5_mag << lz; + wire signed [11:0] bexp_n = 12'sd136 - 12'($signed({1'b0, lz})) + 12'(s5_exp); + `UNUSED_VAR (norm[14:0]) + + wire s6_special, s6_neg, s6_zsign, s6_nz; + wire [31:0] s6_sval; + wire signed [11:0] s6_bexp; + wire [24:0] s6_sig; + + VX_pipe_register #( + .DATAW (1 + 32 + 1 + 1 + 1 + 12 + 25) + ) s6_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s5_special, s5_sval, s5_neg, s5_zsign, mag_nz, bexp_n, norm[39:15]}), + .data_out ({s6_special, s6_sval, s6_neg, s6_zsign, s6_nz, s6_bexp, s6_sig}) + ); + + wire [24:0] sig_r = {1'b0, s6_sig[24:1]} + 25'(s6_sig[0]); + wire signed [11:0] bexp = s6_bexp + 12'(sig_r[24]); + wire [22:0] mant = sig_r[24] ? sig_r[23:1] : sig_r[22:0]; + + reg [31:0] packed_res; + always @(*) begin + if (s6_special) begin + packed_res = s6_sval; + end else if (~s6_nz) begin + packed_res = {s6_zsign, 31'd0}; + end else if (bexp >= 12'sd255) begin + packed_res = {s6_neg, F32_POS_INF[30:0]}; + end else if (bexp <= 12'sd0) begin + packed_res = {s6_neg, 31'd0}; + end else begin + packed_res = {s6_neg, bexp[7:0], mant}; + end + end + + VX_pipe_register #( + .DATAW (32) + ) s7_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (packed_res), + .data_out (result) + ); + +endmodule diff --git a/hw/rtl/mufu/VX_mufu_rom.sv b/hw/rtl/mufu/VX_mufu_rom.sv new file mode 100644 index 0000000000..28b86a76f1 --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_rom.sv @@ -0,0 +1,559 @@ +`include "VX_platform.vh" + +module VX_mufu_rom ( + input wire clk, + input wire enable, + input wire [2:0] tbl, + input wire [6:0] idx, + output wire [31:0] a0, + output wire signed [26:0] a1, + output wire signed [19:0] a2 +); + + reg [8:0] base; + + always @(*) begin + case (tbl) + 3'd0: base = 9'd0; + 3'd1: base = 9'd128; + 3'd2: base = 9'd256; + 3'd3: base = 9'd320; + 3'd4: base = 9'd384; + 3'd5: base = 9'd448; + default: base = '0; + endcase + end + + wire [8:0] addr = base + 9'(idx); + + (* rom_style = "block" *) reg [71:0] mem [0:511]; + + initial begin + mem[0] = 72'h800000070001b07e9f; + mem[1] = 72'h7f01fbef03f6487b9b; + mem[2] = 72'h7e07e06707d31078c9; + mem[3] = 72'h7d1196670b9950760d; + mem[4] = 72'h7c1f07af0f49a87366; + mem[5] = 72'h7b301eb712e4d070d3; + mem[6] = 72'h7a44c69f166b586e54; + mem[7] = 72'h795ceb0f19ddf06be7; + mem[8] = 72'h787878671d3d18698c; + mem[9] = 72'h77975b7f2089706743; + mem[10] = 72'h76b981c723c380650b; + mem[11] = 72'h75ded94726ebc862e2; + mem[12] = 72'h750750672a02d060ca; + mem[13] = 72'h7432d6372d09105ec0; + mem[14] = 72'h73615a172fff085cc5; + mem[15] = 72'h7292cc0732e5285ad8; + mem[16] = 72'h71c71c6735bbd858f8; + mem[17] = 72'h70fe3bf73883985726; + mem[18] = 72'h70381c073b3cc05560; + mem[19] = 72'h6f74ae173de7b853a7; + mem[20] = 72'h6eb3e4474084e851f9; + mem[21] = 72'h6df5b0e74314a85057; + mem[22] = 72'h6d3a06cf4597504ebf; + mem[23] = 72'h6c80d8f7480d484d33; + mem[24] = 72'h6bca1ae74a76d84bb1; + mem[25] = 72'h6b15c0674cd4584a38; + mem[26] = 72'h6a63bd774f261048ca; + mem[27] = 72'h69b40697516c584765; + mem[28] = 72'h6906906753a7784608; + mem[29] = 72'h685b4fdf55d7b844b5; + mem[30] = 72'h67b23a4f57fd58436a; + mem[31] = 72'h670b45375a18a84228; + mem[32] = 72'h666666675c29e040ed; + mem[33] = 72'h65c393df5e31403fba; + mem[34] = 72'h6522c3ef602f103e8f; + mem[35] = 72'h6483ed276223803d6b; + mem[36] = 72'h63e70637640ed83c4e; + mem[37] = 72'h634c063765f1403b38; + mem[38] = 72'h62b2e43f67caf83a28; + mem[39] = 72'h621b97bf699c38391f; + mem[40] = 72'h6186185f6b6530381c; + mem[41] = 72'h60f25de76d26103720; + mem[42] = 72'h6060605f6edf083629; + mem[43] = 72'h5fd017ef7090483538; + mem[44] = 72'h5f417d07723a00344c; + mem[45] = 72'h5eb4881f73dc603366; + mem[46] = 72'h5e2932077577883285; + mem[47] = 72'h5d9f738f770bb031a9; + mem[48] = 72'h5d1745cf7898f830d2; + mem[49] = 72'h5c90a1ff7a1f803000; + mem[50] = 72'h5c0b816f7b9f802f33; + mem[51] = 72'h5b87ddaf7d19182e6a; + mem[52] = 72'h5b05b0577e8c682da6; + mem[53] = 72'h5a84f3477ff9902ce6; + mem[54] = 72'h5a05a0578160b82c2a; + mem[55] = 72'h5987b1a782c2002b72; + mem[56] = 72'h590b2167841d902abe; + mem[57] = 72'h588fe9d78573802a0e; + mem[58] = 72'h5816058786c3e82962; + mem[59] = 72'h579d6ee7880ef828b9; + mem[60] = 72'h572620b78954b82814; + mem[61] = 72'h56b015af8a95582772; + mem[62] = 72'h563b48c78bd0e826d4; + mem[63] = 72'h55c7b4f78d07882639; + mem[64] = 72'h555555578e395825a2; + mem[65] = 72'h54e425278f6660250d; + mem[66] = 72'h54741faf908ec8247c; + mem[67] = 72'h5405405791b2a023ed; + mem[68] = 72'h5397829f92d2082361; + mem[69] = 72'h532ae21f93ed1022d8; + mem[70] = 72'h52bf5a879503d02252; + mem[71] = 72'h5254e78f96166021cf; + mem[72] = 72'h51eb851f9724d8214e; + mem[73] = 72'h51832f27982f4020d0; + mem[74] = 72'h511be1979935c02054; + mem[75] = 72'h50b5989f9a38581fda; + mem[76] = 72'h505050579b37281f63; + mem[77] = 72'h4fec04ff9c32481eef; + mem[78] = 72'h4f88b2f79d29b81e7c; + mem[79] = 72'h4f2656979e1d981e0c; + mem[80] = 72'h4ec4ec4f9f0df81d9e; + mem[81] = 72'h4e6470b79ffae01d32; + mem[82] = 72'h4e04e057a0e4681cc8; + mem[83] = 72'h4da637d7a1caa81c60; + mem[84] = 72'h4d4873f7a2ada01bfa; + mem[85] = 72'h4ceb916fa38d701b95; + mem[86] = 72'h4c8f8d2fa46a181b33; + mem[87] = 72'h4c346407a543b01ad3; + mem[88] = 72'h4bda12ffa61a401a74; + mem[89] = 72'h4b809707a6ede01a17; + mem[90] = 72'h4b27ed37a7be9819bc; + mem[91] = 72'h4ad012b7a88c701962; + mem[92] = 72'h4a7904afa95780190a; + mem[93] = 72'h4a22c04faa1fd018b4; + mem[94] = 72'h49cd42e7aae570185f; + mem[95] = 72'h497889c7aba860180b; + mem[96] = 72'h4924924fac68b817b9; + mem[97] = 72'h48d159e7ad26881769; + mem[98] = 72'h487ede0fade1c8171a; + mem[99] = 72'h482d1c37ae9a9816cc; + mem[100] = 72'h47dc11f7af51001680; + mem[101] = 72'h478bbcf7b004f81635; + mem[102] = 72'h473c1abfb0b6a015eb; + mem[103] = 72'h46ed2907b1660015a3; + mem[104] = 72'h469ee587b21318155c; + mem[105] = 72'h46514e07b2bdf01516; + mem[106] = 72'h4604604fb366a014d1; + mem[107] = 72'h45b81a27b40d28148e; + mem[108] = 72'h456c7987b4b190144b; + mem[109] = 72'h45217c3fb553e8140a; + mem[110] = 72'h44d72047b5f43813c9; + mem[111] = 72'h448d63a7b69280138a; + mem[112] = 72'h44444447b72ed8134c; + mem[113] = 72'h43fbc047b7c938130f; + mem[114] = 72'h43b3d5b7b861b012d3; + mem[115] = 72'h436c82a7b8f8401298; + mem[116] = 72'h4325c547b98d00125e; + mem[117] = 72'h42df9bb7ba1ff01224; + mem[118] = 72'h429a042fbab11011ec; + mem[119] = 72'h4254fce7bb407011b5; + mem[120] = 72'h42108427bbce10117e; + mem[121] = 72'h41cc982fbc5a081149; + mem[122] = 72'h41893757bce4481114; + mem[123] = 72'h41465fe7bd6ce810e0; + mem[124] = 72'h41041047bdf3e810ad; + mem[125] = 72'h40c246dfbe7948107a; + mem[126] = 72'h40810207befd201049; + mem[127] = 72'h40404047bf7f601018; + mem[128] = 72'h80000007000438bc93; + mem[129] = 72'h7f02f5df05e7c0b535; + mem[130] = 72'h7e0bb1e70b9128ae78; + mem[131] = 72'h7d19fc671104b8a814; + mem[132] = 72'h7c2da0ef164528a203; + mem[133] = 72'h7b466da71b55109c3f; + mem[134] = 72'h7a6433372036e096c4; + mem[135] = 72'h7986c4b724ecd0918c; + mem[136] = 72'h78adf74f2979088c95; + mem[137] = 72'h77d9a2472ddd9087d9; + mem[138] = 72'h77099ed7321c308356; + mem[139] = 72'h763dc7ff3636c07f07; + mem[140] = 72'h7575fa873a2ed87aeb; + mem[141] = 72'h74b214af3e061876fe; + mem[142] = 72'h73f1f66f41bde8733d; + mem[143] = 72'h733580ff4557b06fa6; + mem[144] = 72'h727c96ff48d4c86c36; + mem[145] = 72'h71c71c574c366868ec; + mem[146] = 72'h7114f62f4f7db065c6; + mem[147] = 72'h70660aaf52abd062c1; + mem[148] = 72'h6fba414755c1c05fdb; + mem[149] = 72'h6f11823758c0885d14; + mem[150] = 72'h6e6bb6d75ba9185a69; + mem[151] = 72'h6dc8c95f5e7c5057da; + mem[152] = 72'h6d28a4df613b105564; + mem[153] = 72'h6c8b354763e6205306; + mem[154] = 72'h6bf06757667e4050c0; + mem[155] = 72'h6b5828676904384e90; + mem[156] = 72'h6ac266a76b78a84c75; + mem[157] = 72'h6a2f10f76ddc404a6d; + mem[158] = 72'h699e16c7702fa04879; + mem[159] = 72'h690f681f7273604697; + mem[160] = 72'h6882f5b774a81044c7; + mem[161] = 72'h67f8b0b776ce404307; + mem[162] = 72'h67708aef78e6684157; + mem[163] = 72'h66ea768f7af1183fb6; + mem[164] = 72'h6666665f7ceeb83e23; + mem[165] = 72'h65e44d877edfc83c9e; + mem[166] = 72'h65641fa780c4b03b26; + mem[167] = 72'h64e5d0d7829de039bb; + mem[168] = 72'h6469557f846bb0385c; + mem[169] = 72'h63eea27f862e883709; + mem[170] = 72'h6375ad0f87e6c835c0; + mem[171] = 72'h62fe6abf8994c03482; + mem[172] = 72'h6288d16f8b38d0334f; + mem[173] = 72'h6214d75f8cd3403225; + mem[174] = 72'h61a2731f8e64603105; + mem[175] = 72'h61319b778fec802fed; + mem[176] = 72'h60c24797916be82ede; + mem[177] = 72'h60546edf92e2d02dd8; + mem[178] = 72'h5fe808ff9451882cd9; + mem[179] = 72'h5f7d0dd795b8502be2; + mem[180] = 72'h5f1375979717582af3; + mem[181] = 72'h5eab38a7986ef02a0a; + mem[182] = 72'h5e444faf99bf382929; + mem[183] = 72'h5ddeb3779b0880284e; + mem[184] = 72'h5d7a5d179c4ae82779; + mem[185] = 72'h5d1745d79d86a826aa; + mem[186] = 72'h5cb567179ebbf825e1; + mem[187] = 72'h5c54ba7f9feb00251e; + mem[188] = 72'h5bf539e7a113f02461; + mem[189] = 72'h5b96df47a236f023a8; + mem[190] = 72'h5b39a4c7a3543022f5; + mem[191] = 72'h5add84bfa46bd02246; + mem[192] = 72'h5a82796f4afef08528; + mem[193] = 72'h59cf8c8f4f28088022; + mem[194] = 72'h5920b4b75328e87b5e; + mem[195] = 72'h5875cab75703b876da; + mem[196] = 72'h57cea9775aba60728f; + mem[197] = 72'h572b2dbf5e4eb86e7c; + mem[198] = 72'h568b360f61c2786a9b; + mem[199] = 72'h55eea2a765173066eb; + mem[200] = 72'h55555537684e706368; + mem[201] = 72'h54bf31076b6990600f; + mem[202] = 72'h542c1a8f6e69f05cde; + mem[203] = 72'h539bf7b77150d059d3; + mem[204] = 72'h530eaf8f741f5056ea; + mem[205] = 72'h52842a4776d6905424; + mem[206] = 72'h51fc512f797798517c; + mem[207] = 72'h51770e7f7c03684ef2; + mem[208] = 72'h50f44d777e7ae84c85; + mem[209] = 72'h5073fa4780def84a31; + mem[210] = 72'h4ff601cf83307847f7; + mem[211] = 72'h4f7a51f785702045d4; + mem[212] = 72'h4f00d937879eb043c8; + mem[213] = 72'h4e8986d789bce841d1; + mem[214] = 72'h4e144adf8bcb603fee; + mem[215] = 72'h4da115cf8dcac83e1f; + mem[216] = 72'h4d2fd8e78fbbb03c61; + mem[217] = 72'h4cc085f7919eb03ab5; + mem[218] = 72'h4c530f579374503919; + mem[219] = 72'h4be767ef953d10378d; + mem[220] = 72'h4b7d830f96f9703610; + mem[221] = 72'h4b15549f98a9e834a1; + mem[222] = 72'h4aaed0ef9a4ee0333f; + mem[223] = 72'h4a49ecaf9be8d831ea; + mem[224] = 72'h49e69d179d782030a2; + mem[225] = 72'h4984d79f9efd282f65; + mem[226] = 72'h49249247a078502e34; + mem[227] = 72'h48c5c347a1e9e02d0d; + mem[228] = 72'h48686147a352402bf0; + mem[229] = 72'h480c632fa4b1b82add; + mem[230] = 72'h47b1c047a608a029d3; + mem[231] = 72'h47587017a7573028d2; + mem[232] = 72'h47006a7fa89dc027da; + mem[233] = 72'h46a9a797a9dc8826ea; + mem[234] = 72'h46541fb7ab13d82602; + mem[235] = 72'h45ffcb7fac43e02521; + mem[236] = 72'h45aca3d7ad6ce82448; + mem[237] = 72'h455aa1cfae8f202375; + mem[238] = 72'h4509beafafaac822a9; + mem[239] = 72'h44b9f407b0c01021e4; + mem[240] = 72'h446b3b97b1cf282124; + mem[241] = 72'h441d8f3fb2d840206a; + mem[242] = 72'h43d0e917b3db901fb6; + mem[243] = 72'h43854377b4d9401f08; + mem[244] = 72'h433a98c7b5d1801e5e; + mem[245] = 72'h42f0e3afb6c4701dba; + mem[246] = 72'h42a81ef7b7b2381d1b; + mem[247] = 72'h4260458fb89b101c80; + mem[248] = 72'h4219528fb97f081be9; + mem[249] = 72'h41d34127ba5e501b57; + mem[250] = 72'h418e0ccfbb39001ac9; + mem[251] = 72'h4149b0e7bc0f481a3f; + mem[252] = 72'h41062927bce14019b9; + mem[253] = 72'h40c3713fbdaf081937; + mem[254] = 72'h40818517be78b818b8; + mem[255] = 72'h404060a7bf3e78183c; + mem[256] = 72'h8000000162e3687b9e; + mem[257] = 72'h8164d23166c0807d03; + mem[258] = 72'h82cd86d96aa8907e5f; + mem[259] = 72'h843a29016e9b887fc0; + mem[260] = 72'h85aac3a97299808124; + mem[261] = 72'h871f61d976a2a0828c; + mem[262] = 72'h88980ec17ab6f883f8; + mem[263] = 72'h8a14d5b97ed6b08567; + mem[264] = 72'h8b95c2298301e886db; + mem[265] = 72'h8d1adfa18738c08853; + mem[266] = 72'h8ea439d18b7b5889cf; + mem[267] = 72'h9031dc898fc9d08b4f; + mem[268] = 72'h91c3d3b99424488cd4; + mem[269] = 72'h935a2b79988ae88e5c; + mem[270] = 72'h94f4eff19cfdc88fe9; + mem[271] = 72'h96942d81a17d10917a; + mem[272] = 72'h9837f099a608e09310; + mem[273] = 72'h99e045d9aaa16094aa; + mem[274] = 72'h9b8d3a01af46a89648; + mem[275] = 72'h9d3ed9f1b3f8e897eb; + mem[276] = 72'h9ef532b1b8b8409993; + mem[277] = 72'ha0b05161bd84d09b3f; + mem[278] = 72'ha2704351c25ec89cf0; + mem[279] = 72'ha4351601c746409ea5; + mem[280] = 72'ha5fed6f9cc3b68a05f; + mem[281] = 72'ha7cd9409d13e60a21e; + mem[282] = 72'ha9a15b09d64f50a3e2; + mem[283] = 72'hab7a3a09db6e60a5ab; + mem[284] = 72'had583f41e09bb8a779; + mem[285] = 72'haf3b7901e5d780a94c; + mem[286] = 72'hb123f5d9eb21d8ab24; + mem[287] = 72'hb311c469f07af8ad01; + mem[288] = 72'hb504f389f5e2f8aee3; + mem[289] = 72'hb6fd9239fb5a10b0cb; + mem[290] = 72'hb8fbafa200e060b2b8; + mem[291] = 72'hbaff5b0a067620b4aa; + mem[292] = 72'hbd08a3fa0c1b68b6a1; + mem[293] = 72'hbf179a1211d070b89f; + mem[294] = 72'hc12c4d2a179560baa1; + mem[295] = 72'hc346cd3a1d6a68bca9; + mem[296] = 72'hc5672a72234fb0beb7; + mem[297] = 72'hc78d752a294568c0cb; + mem[298] = 72'hc9b9bdea2f4bb8c2e4; + mem[299] = 72'hcbec15623562d8c504; + mem[300] = 72'hce248c7a3b8af0c729; + mem[301] = 72'hd063344241c438c954; + mem[302] = 72'hd2a81df2480ed8cb85; + mem[303] = 72'hd4f35b124e6b00cdbd; + mem[304] = 72'hd744fd3254d8e0cffa; + mem[305] = 72'hd99d162a5b58b0d23e; + mem[306] = 72'hdbfbb80261ea98d488; + mem[307] = 72'hde60f4ea688ed8d6d9; + mem[308] = 72'he0ccdf5a6f4598d92f; + mem[309] = 72'he33f89e2760f10db8d; + mem[310] = 72'he5b907527ceb70ddf1; + mem[311] = 72'he8396ac283daf8e05c; + mem[312] = 72'heac0c75a8addd0e2cd; + mem[313] = 72'hed4f309291f430e545; + mem[314] = 72'hefe4ba12991e58e7c4; + mem[315] = 72'hf28177b2a05c78ea4b; + mem[316] = 72'hf5257d8aa7aec8ecd8; + mem[317] = 72'hf7d0dfeaaf1580ef6c; + mem[318] = 72'hfa83b352b690d8f207; + mem[319] = 72'hfd3e0c82be2110f4aa; + mem[320] = 72'h00000002e2a27693f4; + mem[321] = 72'h02dcf34ad743a69f61; + mem[322] = 72'h05aeb552cc3efea9e8; + mem[323] = 72'h08759cbac18e86b3f7; + mem[324] = 72'h0b31fbe2b72e86bd97; + mem[325] = 72'h0de42182ad1b7ec6cd; + mem[326] = 72'h108c58eaa35226cfa0; + mem[327] = 72'h132aea4299cf5ed814; + mem[328] = 72'h15c01a92909036e030; + mem[329] = 72'h184c2c228791e6e7f7; + mem[330] = 72'h1acf5e827ed1ceef6f; + mem[331] = 72'h1d49ee9a764d6ef69b; + mem[332] = 72'h1fbc17026e0276fd80; + mem[333] = 72'h2226100265ee970420; + mem[334] = 72'h24880f9a5e0fc70a81; + mem[335] = 72'h26e249e25663ef10a4; + mem[336] = 72'h2934f0da4ee937168d; + mem[337] = 72'h2b8034b2479dbf1c3f; + mem[338] = 72'h2dc443da407fcf21bb; + mem[339] = 72'h30014b02398dc72706; + mem[340] = 72'h3237755232c6172c21; + mem[341] = 72'h3466ec522c273f310e; + mem[342] = 72'h368fd82225afcf35d0; + mem[343] = 72'h38b25f921f5e673a69; + mem[344] = 72'h3acea7f21931c73ed9; + mem[345] = 72'h3ce4d5721328a74324; + mem[346] = 72'h3ef50b020d41df474b; + mem[347] = 72'h40ff6a5a077c4f4b4f; + mem[348] = 72'h4304143201d6d74f31; + mem[349] = 72'h45032811fc507752f4; + mem[350] = 72'h46fcc4a1f6e82f5699; + mem[351] = 72'h48f10779f19d075a21; + mem[352] = 72'h4ae00d41ec6e1f5d8c; + mem[353] = 72'h4cc9f1d1e75a8f60dd; + mem[354] = 72'h4eaed011e261876414; + mem[355] = 72'h508ec221dd822f6732; + mem[356] = 72'h5269e151d8bbcf6a39; + mem[357] = 72'h54404641d40da76d29; + mem[358] = 72'h561208c1cf76f77002; + mem[359] = 72'h57df3ff1caf71772c7; + mem[360] = 72'h59a80259c68d5f7578; + mem[361] = 72'h5b6c65c9c239277815; + mem[362] = 72'h5d2c7f79bdf9df7a9f; + mem[363] = 72'h5ee86401b9cee77d18; + mem[364] = 72'h60a02771b5b7af7f7f; + mem[365] = 72'h6253dd49b1b3af81d5; + mem[366] = 72'h64039871adc25f841b; + mem[367] = 72'h65af6b61a9e33f8651; + mem[368] = 72'h67576811a615cf8878; + mem[369] = 72'h68fb9fe9a2599f8a91; + mem[370] = 72'h6a9c23f19eae2f8c9c; + mem[371] = 72'h6c3904b19b13178e99; + mem[372] = 72'h6dd252519787ef908a; + mem[373] = 72'h6f681c91940c3f926d; + mem[374] = 72'h70fa72a1909fb79445; + mem[375] = 72'h728963898d41df9610; + mem[376] = 72'h7414fdc989f26f97d1; + mem[377] = 72'h759d4f9186b0f79986; + mem[378] = 72'h772266c1837d2f9b30; + mem[379] = 72'h78a450c98056b79cd1; + mem[380] = 72'h7a231ae17d3d3f9e67; + mem[381] = 72'h7b9ed1d97a307f9ff4; + mem[382] = 72'h7d17824177301fa177; + mem[383] = 72'h7e8d3859743bdfa2f1; + mem[384] = 72'h00000003244857f8cf; + mem[385] = 72'h03242a83240d0fe8c0; + mem[386] = 72'h0647d94323530fd944; + mem[387] = 72'h096a900b221d2fc9ce; + mem[388] = 72'h0c8bd323206b9fba61; + mem[389] = 72'h0fab26eb1e3e9faafe; + mem[390] = 72'h12c810331b96879ba8; + mem[391] = 72'h15e214031873bf8c62; + mem[392] = 72'h18f8b7fb14d6c77d2e; + mem[393] = 72'h1c0b822b10c0276e0e; + mem[394] = 72'h1f19f93b0c30875f04; + mem[395] = 72'h2223a48b07288f5013; + mem[396] = 72'h25280c2301a917413d; + mem[397] = 72'h2826b8f2fbb2ef3285; + mem[398] = 72'h2b1f34b2f5470723ec; + mem[399] = 72'h2e110a2aee664f1576; + mem[400] = 72'h30fbc512e711e70723; + mem[401] = 72'h33def252df4ae6f8f7; + mem[402] = 72'h36ba1fdad71286eaf3; + mem[403] = 72'h398cdd02ce69fedd1a; + mem[404] = 72'h3c56ba3ac552b6cf6e; + mem[405] = 72'h3f174982bbce0ec1f1; + mem[406] = 72'h41ce1e32b1dd76b4a5; + mem[407] = 72'h447acd22a7827ea78c; + mem[408] = 72'h471cecb29cbec69aa9; + mem[409] = 72'h49b415029193e68dfc; + mem[410] = 72'h4c3fdfc286039e8188; + mem[411] = 72'h4ebfe8727a0fbe7550; + mem[412] = 72'h5133cc626dba0e6954; + mem[413] = 72'h539b2ac26104865d97; + mem[414] = 72'h55f5a4a253f116521a; + mem[415] = 72'h5842dd224681be46e0; + mem[416] = 72'h5a82797238b88e3bea; + mem[417] = 72'h5cb420b22a97ae3139; + mem[418] = 72'h5ed77c621c214626d0; + mem[419] = 72'h60ec38020d579e1cb0; + mem[420] = 72'h62f20189fe3ce612da; + mem[421] = 72'h64e88901eed3860950; + mem[422] = 72'h66cf8101df1dd60014; + mem[423] = 72'h68a69e61cf1e45f727; + mem[424] = 72'h6a6d9881bed74dee8a; + mem[425] = 72'h6c242941ae4b6de63f; + mem[426] = 72'h6dca0cf19d7d2dde47; + mem[427] = 72'h6f5f02918c6f2dd6a3; + mem[428] = 72'h70e2cba97b240dcf54; + mem[429] = 72'h72552c69699e75c85b; + mem[430] = 72'h73b5ebb157e11dc1bb; + mem[431] = 72'h7504d33145eeb5bb72; + mem[432] = 72'h7641af2933ca0db584; + mem[433] = 72'h776c4ec92175f5afef; + mem[434] = 72'h788484010ef535aab6; + mem[435] = 72'h798a23a0fc4aada5d9; + mem[436] = 72'h7a7d0548e9793da159; + mem[437] = 72'h7b5d0390d683cd9d37; + mem[438] = 72'h7c29fbe0c36d4d9973; + mem[439] = 72'h7ce3cea0b038a5960d; + mem[440] = 72'h7d8a5f389ce8cd9307; + mem[441] = 72'h7e1d93e08980cd9060; + mem[442] = 72'h7e9d55f07603958e1a; + mem[443] = 72'h7f0991c06274258c34; + mem[444] = 72'h7f6236884ed58d8aaf; + mem[445] = 72'h7fa736b03b2ac5898b; + mem[446] = 72'h7fd887882776e588c9; + mem[447] = 72'h7ff6218013bced8867; + mem[448] = 72'h07ff5588ffc03ffde1; + mem[449] = 72'h08ff0d40ffaf47fda1; + mem[450] = 72'h09feb300ff9c4ffd62; + mem[451] = 72'h0afe44c8ff875ffd22; + mem[452] = 72'h0bfdc0a0ff706ffce3; + mem[453] = 72'h0cfd2490ff577ffca3; + mem[454] = 72'h0dfc6e90ff3c97fc64; + mem[455] = 72'h0efb9cb0ff1fb7fc25; + mem[456] = 72'h07fd5680ff016ff78d; + mem[457] = 72'h08fc3600febdd7f692; + mem[458] = 72'h09face00fe725ff598; + mem[459] = 72'h0af916a8fe1f1ff4a0; + mem[460] = 72'h0bf70828fdc41ff3a8; + mem[461] = 72'h0cf49ac8fd615ff2b3; + mem[462] = 72'h0df1c6d8fcf6f7f1bf; + mem[463] = 72'h0eee84b0fc84e7f0cd; + mem[464] = 72'h07f56640fc0d67deca; + mem[465] = 72'h08f0ee78fb03afdb19; + mem[466] = 72'h09eb5e38f9dc67d77a; + mem[467] = 72'h0ae49838f89827d3ee; + mem[468] = 72'h0bdc7fc0f7378fd077; + mem[469] = 72'h0cd2f8d8f5bb37cd17; + mem[470] = 72'h0dc7e820f423dfc9ce; + mem[471] = 72'h0ebb32e0f2723fc69f; + mem[472] = 72'h07d65f48f0add78416; + mem[473] = 72'h08c51c78eccdf778a0; + mem[474] = 72'h09afcc00e8925f6e17; + mem[475] = 72'h0a9615e8e402776485; + mem[476] = 72'h0b77a9b8df25ff5bf3; + mem[477] = 72'h0c543ee0da04f75465; + mem[478] = 72'h0d2b94d0d4a7774ddb; + mem[479] = 72'h0dfd7340cf15af4856; + mem[480] = 72'h0764d478c95fce83fb; + mem[481] = 72'h08284348bd7b0e7982; + mem[482] = 72'h08dfa408b1430e7603; + mem[483] = 72'h098abf50a4ef9e78ba; + mem[484] = 72'h0a29927898b26e80b0; + mem[485] = 72'h0abc48a08cb58e8cf6; + mem[486] = 72'h0b433350811b669c9c; + mem[487] = 72'h0bbec2a075fefeaec1; + mem[488] = 72'h0617c2406b453d99fc; + mem[489] = 72'h0679767058143deeb1; + mem[490] = 72'h06c94c50478cb64190; + mem[491] = 72'h0709e5a8399e3e8de0; + mem[492] = 72'h073dc1082e1336d146; + mem[493] = 72'h07671e2824a3770b06; + mem[494] = 72'h0787f1c01d014f3b68; + mem[495] = 72'h07a1e40016e18f6344; + mem[496] = 72'h03db2e7011c4d72203; + mem[497] = 72'h03e982780aecbf7643; + mem[498] = 72'h03f24cc006ae6fab4b; + mem[499] = 72'h03f7ab280412afcc31; + mem[500] = 72'h03faf050027a5fe06a; + mem[501] = 72'h03fced6001817fecc8; + mem[502] = 72'h03fe229800ea17f453; + mem[503] = 72'h03fede58008e17f8e9; + mem[504] = 72'h01ffa86000525ff92a; + mem[505] = 72'h01ffdfc0001e57fd7c; + mem[506] = 72'h01fff420000b2fff13; + mem[507] = 72'h01fffba800041fffa9; + mem[508] = 72'h01fffe68000187ffe0; + mem[509] = 72'h01ffff68000097fff4; + mem[510] = 72'h01ffffc800003ffffc; + mem[511] = 72'h01fffff0000017fffe; + end + + reg [71:0] rd_data, rd_data_r; + + always @(posedge clk) begin + if (enable) begin + rd_data <= mem[addr]; + rd_data_r <= rd_data; + end + end + + assign a0 = 32'({rd_data_r[71:43], 2'b0}); + assign a1 = 27'($signed({rd_data_r[42:19], 2'b0})); + assign a2 = 20'($signed({rd_data_r[18:0], 1'b0})); + +endmodule diff --git a/hw/rtl/mufu/VX_mufu_unit.sv b/hw/rtl/mufu/VX_mufu_unit.sv new file mode 100644 index 0000000000..5e390fe1d4 --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_unit.sv @@ -0,0 +1,127 @@ +`include "VX_define.vh" + +module VX_mufu_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], + + VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH] +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam BLOCK_SIZE = `VX_CFG_NUM_MUFU_BLOCKS; + localparam NUM_LANES = `VX_CFG_NUM_MUFU_LANES; + localparam LATENCY = `VX_CFG_MUFU_LATENCY; + localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); + localparam HDR_WIDTH = $bits(mufu_header_t); + + VX_execute_if #( + .data_t (mufu_execute_t) + ) per_block_execute_if[BLOCK_SIZE](); + + VX_lane_dispatch #( + .BLOCK_SIZE (BLOCK_SIZE), + .NUM_LANES (NUM_LANES), + .OUT_BUF (PARTIAL_BW ? 3 : 0) + ) lane_dispatch ( + .clk (clk), + .reset (reset), + .dispatch_if(dispatch_if), + .execute_if (per_block_execute_if) + ); + + VX_result_if #( + .data_t (mufu_result_t) + ) per_block_result_if[BLOCK_SIZE](); + + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + wire [NUM_LANES-1:0][31:0] req_data; + wire [NUM_LANES-1:0][31:0] rsp_data; + wire [NUM_LANES-1:0][31:0] pe_data_in; + wire [NUM_LANES-1:0][31:0] pe_data_out; + wire [2:0] pe_fn; + wire pe_enable; + mufu_header_t rsp_hdr; + + `UNUSED_VAR (per_block_execute_if[block_idx].data.op_args) + `UNUSED_VAR (per_block_execute_if[block_idx].data.rs2_data) + `UNUSED_VAR (per_block_execute_if[block_idx].data.rs3_data) + `UNUSED_VAR (per_block_execute_if[block_idx].data.op_type) + + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_req_data + wire [`VX_CFG_XLEN-1:0] rs1 = per_block_execute_if[block_idx].data.rs1_data[i]; + if (`VX_CFG_XLEN > 32) begin : g_unbox + assign req_data[i] = (&rs1[`VX_CFG_XLEN-1:32]) ? rs1[31:0] : 32'h7fc00000; + end else begin : g_nobox + assign req_data[i] = rs1[31:0]; + end + end + + VX_pe_serializer #( + .NUM_LANES (NUM_LANES), + .NUM_PES (NUM_LANES), + .LATENCY (LATENCY), + .DATA_IN_WIDTH (32), + .DATA_OUT_WIDTH (32), + .SHARED_WIDTH (3), + .TAG_WIDTH (HDR_WIDTH), + .PE_REG (0), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) pe_serializer ( + .clk (clk), + .reset (reset), + .valid_in (per_block_execute_if[block_idx].valid), + .mask_in (per_block_execute_if[block_idx].data.header.tmask), + .data_in (req_data), + .shared_in (per_block_execute_if[block_idx].data.op_type[2:0]), + .tag_in (per_block_execute_if[block_idx].data.header), + .ready_in (per_block_execute_if[block_idx].ready), + .pe_enable (pe_enable), + `UNUSED_PIN (pe_mask_out), + .pe_data_out(pe_data_in), + .pe_shared_out(pe_fn), + .pe_data_in (pe_data_out), + .valid_out (per_block_result_if[block_idx].valid), + `UNUSED_PIN (mask_out), + .data_out (rsp_data), + .tag_out (rsp_hdr), + .ready_out (per_block_result_if[block_idx].ready) + ); + + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_pes + VX_mufu_pe #( + .LATENCY (LATENCY) + ) pe ( + .clk (clk), + .reset (reset), + .enable (pe_enable), + .fn (pe_fn), + .dataa (pe_data_in[i]), + .result (pe_data_out[i]) + ); + end + + assign per_block_result_if[block_idx].data.header = rsp_hdr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data + if (`VX_CFG_XLEN > 32) begin : g_box + assign per_block_result_if[block_idx].data.data[i] = {{(`VX_CFG_XLEN-32){1'b1}}, rsp_data[i]}; + end else begin : g_nobox + assign per_block_result_if[block_idx].data.data[i] = rsp_data[i]; + end + end + end + + VX_lane_gather #( + .BLOCK_SIZE (BLOCK_SIZE), + .NUM_LANES (NUM_LANES), + .OUT_BUF (PARTIAL_BW ? 3 : 0) + ) lane_gather ( + .clk (clk), + .reset (reset), + .result_if (per_block_result_if), + .commit_if (commit_if) + ); + +endmodule diff --git a/hw/scripts/gen_mufu_tables.py b/hw/scripts/gen_mufu_tables.py new file mode 100644 index 0000000000..954c7c26cf --- /dev/null +++ b/hw/scripts/gen_mufu_tables.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the MUFU coefficient ROM. + +Every table interval is approximated as P = A0 + A1*z + A2*z^2 in 30-bit fixed +point, z = xl/2^18 being the position inside the interval. Evaluation is the +bit-exact integer form shared by sw/common/mufu.cpp and hw/rtl/mufu: + + P = A0 + ((A1*xl) >> 18) + ((A2*((xl*xl) >> 18)) >> 18) + +The ROM stores each coefficient without its S0/S1/S2 low bits (Ak = stored << +Sk), so one entry is 72 bits and the 512 entries fill one 512x72 RAMB36. + +Outputs (checked in, regenerate after any change here): + sw/common/mufu_tables.h + hw/rtl/mufu/VX_mufu_rom.sv (no comments, by project convention for this unit) + +Usage: + gen_mufu_tables.py write both outputs + gen_mufu_tables.py --check fail if the checked-in outputs are stale + gen_mufu_tables.py --report print per-table interpolation error +""" + +import argparse +import math +import os +import sys + +import numpy as np + +FRAC = 30 +XL_BITS = 18 +# Coefficient widths seen by the datapath, and low bits dropped in the ROM. +A0_BITS = 32 +A1_BITS = 27 +A2_BITS = 20 +S0, S1, S2 = 2, 2, 1 +W0, W1, W2 = A0_BITS - 1 - S0, A1_BITS - 1 - S1, A2_BITS - S2 +ROM_DEPTH = 512 + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +OUT_H = os.path.join(ROOT, "sw", "common", "mufu_tables.h") +OUT_SV = os.path.join(ROOT, "hw", "rtl", "mufu", "VX_mufu_rom.sv") + + +# Intervals per table, sized for PTX .approx-level accuracy; --report prints the +# resulting worst-case errors. +N_RCP = 128 # 1/(1+t) +N_RSQRT = 64 # per exponent parity +N_EX2 = 64 +N_LG2 = 64 +N_TRIG = 64 # sin(pi/2 t); cos reads it at 1 - t +TANH_J0 = -5 # first tanh binade (below it tanh x = x) +TANH_BINADES = 8 # binades 2^-5 .. 2^2 (from 8 up tanh x = 1) +TANH_PER = 8 # intervals per tanh binade +TANH_PRESCALE = 4 # tanh entries hold 2^-(j+4)*tanh, keeping A1 in range + + +def _rcp(idx, z): + return 1.0 / (1.0 + (idx + z) / N_RCP) + + +def _rsqrt(idx, z): + t = ((idx % N_RSQRT) + z) / N_RSQRT + s = (1.0 + t) * (2.0 if (idx // N_RSQRT) else 1.0) + return 1.0 / np.sqrt(s) + + +def _ex2(idx, z): + return np.exp2((idx + z) / N_EX2) + + +def _lg2(idx, z): + return np.log2(1.0 + (idx + z) / N_LG2) + + +def _sin(idx, z): + return np.sin(np.pi / 2.0 * (idx + z) / N_TRIG) + + +def _tanh(idx, z): + j = (idx // TANH_PER) + TANH_J0 + t = ((idx % TANH_PER) + z) / TANH_PER + return np.ldexp(np.tanh(np.ldexp(1.0 + t, j)), -j - TANH_PRESCALE) + + +def _mant_step(n): + # xl takes the mantissa bits below the index, left-aligned in XL_BITS. + low = 23 - (n.bit_length() - 1) + return (1 << (XL_BITS - low)) if low < XL_BITS else 1, low > XL_BITS + + +# name, entries, f(idx, z), xl step (xl values are multiples of it), +# xl truncated (true argument lies anywhere in [xl, xl+1)), exact-left intervals +TABLES = [ + ("RCP", N_RCP, _rcp, *_mant_step(N_RCP), {0}), + ("RSQRT", 2 * N_RSQRT, _rsqrt, *_mant_step(N_RSQRT), {0}), + ("EX2", N_EX2, _ex2, 1, True, {0}), + ("LG2", N_LG2, _lg2, *_mant_step(N_LG2), {0}), + ("SIN", N_TRIG, _sin, 1, True, {0}), + ("TANH", TANH_BINADES * TANH_PER, _tanh, *_mant_step(TANH_PER), set()), +] + + +def eval_fixed(a0, a1, a2, xl): + sq = (xl * xl) >> XL_BITS + return a0 + ((a1 * xl) >> XL_BITS) + ((a2 * sq) >> XL_BITS) + + +def minimax(A, y, iters=40): + # Lawson's iteration: reweighted least squares converging to the minimax fit. + w = np.full(len(y), 1.0 / len(y)) + sol = None + for _ in range(iters): + sw = np.sqrt(w) + sol, *_ = np.linalg.lstsq(A * sw[:, None], y * sw, rcond=None) + w = w * (np.abs(A @ sol - y) + 1e-12) + w /= w.sum() + return sol + + +def fit_interval(f, idx, step, trunc, exact_left): + off = 0.5 if trunc else 0.0 + zs = (np.linspace(0, (1 << XL_BITS) - step, 2049) + off) / float(1 << XL_BITS) + ys = f(idx, zs) * (1 << FRAC) + + if exact_left: + a0 = int(round(float(f(idx, 0.0)) * (1 << FRAC))) + a1, a2 = minimax(np.stack([zs, zs * zs], axis=1), ys - a0) + else: + a0, a1, a2 = minimax(np.stack([np.ones_like(zs), zs, zs * zs], axis=1), ys) + a0 = int(round(a0)) + a1 = int(round(a1 / (1 << S1))) << S1 + a2 = int(round(a2 / (1 << S2))) << S2 + + xl = np.arange(0, 1 << XL_BITS, step, dtype=np.int64) + target = f(idx, (xl + off) / float(1 << XL_BITS)) * (1 << FRAC) + if not exact_left: + err = eval_fixed(a0, a1, a2, xl) - target + a0 -= int(round((err.max() + err.min()) / 2.0)) + assert not exact_left or a0 % (1 << S0) == 0, (idx, a0) + a0 = int(round(a0 / (1 << S0))) << S0 + err = eval_fixed(a0, a1, a2, xl) - target + return a0, a1, a2, float(np.abs(err).max()) + + +def build(): + entries = [] + report = [] + bases = [] + base = 0 + for name, n, f, step, trunc, exact in TABLES: + bases.append((name, base, n)) + worst = 0.0 + wid = [0, 0, 0] + for idx in range(n): + a0, a1, a2, e = fit_interval(f, idx, step, trunc, idx in exact) + q0, q1, q2 = a0 >> S0, a1 >> S1, a2 >> S2 + assert 0 <= q0 < (1 << W0), (name, idx, a0) + assert -(1 << (W1 - 1)) <= q1 < (1 << (W1 - 1)), (name, idx, a1) + assert -(1 << (W2 - 1)) <= q2 < (1 << (W2 - 1)), (name, idx, a2) + entries.append((q0, q1, q2)) + worst = max(worst, e) + wid = [max(wid[0], a0.bit_length()), max(wid[1], abs(a1).bit_length() + 1), + max(wid[2], abs(a2).bit_length() + 1)] + report.append((name, worst, n, wid)) + base += n + assert len(entries) <= ROM_DEPTH and W0 + W1 + W2 <= 72 + return entries, bases, report + + +def emit_h(entries, bases): + out = [] + out.append("// Generated by hw/scripts/gen_mufu_tables.py. Do not edit.") + out.append("#pragma once") + out.append("") + out.append("#include ") + out.append("") + out.append("namespace vortex::mufu {") + out.append("") + out.append("// Stored coefficients; the datapath uses a0 << S0, a1 << S1, a2 << S2.") + out.append("struct coeff_t { uint32_t a0; int32_t a1; int32_t a2; };") + out.append("") + out.append(f"constexpr uint32_t S0 = {S0};") + out.append(f"constexpr uint32_t S1 = {S1};") + out.append(f"constexpr uint32_t S2 = {S2};") + out.append("") + for name, base, n in bases: + out.append(f"constexpr uint32_t TBL_{name}_BASE = {base};") + out.append(f"constexpr uint32_t TBL_SIZE = {len(entries)};") + out.append("") + out.append("constexpr coeff_t kTable[TBL_SIZE] = {") + for a0, a1, a2 in entries: + out.append(f" {{0x{a0:08x}u, {a1}, {a2}}},") + out.append("};") + out.append("") + out.append("} // namespace vortex::mufu") + out.append("") + return "\n".join(out) + + +def emit_sv(entries, bases): + aw = (ROM_DEPTH - 1).bit_length() + iw = max(1, (max(n for _, _, n in bases) - 1).bit_length()) + tw = max(1, (len(bases) - 1).bit_length()) + w = W0 + W1 + W2 + out = [] + out.append("`include \"VX_platform.vh\"") + out.append("") + out.append("module VX_mufu_rom (") + out.append(" input wire clk,") + out.append(" input wire enable,") + out.append(f" input wire [{tw - 1}:0] tbl,") + out.append(f" input wire [{iw - 1}:0] idx,") + out.append(f" output wire [{A0_BITS - 1}:0] a0,") + out.append(f" output wire signed [{A1_BITS - 1}:0] a1,") + out.append(f" output wire signed [{A2_BITS - 1}:0] a2") + out.append(");") + out.append("") + out.append(f" reg [{aw - 1}:0] base;") + out.append("") + out.append(" always @(*) begin") + out.append(" case (tbl)") + for i, (_, base, _) in enumerate(bases): + out.append(f" {tw}'d{i}: base = {aw}'d{base};") + out.append(f" default: base = '0;") + out.append(" endcase") + out.append(" end") + out.append("") + out.append(f" wire [{aw - 1}:0] addr = base + {aw}'(idx);") + out.append("") + out.append(f" (* rom_style = \"block\" *) reg [{w - 1}:0] mem [0:{ROM_DEPTH - 1}];") + out.append("") + out.append(" initial begin") + for i in range(ROM_DEPTH): + q0, q1, q2 = entries[i] if i < len(entries) else (0, 0, 0) + v = (q0 << (W1 + W2)) | ((q1 & ((1 << W1) - 1)) << W2) | (q2 & ((1 << W2) - 1)) + out.append(f" mem[{i}] = {w}'h{v:0{(w + 3) // 4}x};") + out.append(" end") + out.append("") + out.append(f" reg [{w - 1}:0] rd_data, rd_data_r;") + out.append("") + out.append(" always @(posedge clk) begin") + out.append(" if (enable) begin") + out.append(" rd_data <= mem[addr];") + out.append(" rd_data_r <= rd_data;") + out.append(" end") + out.append(" end") + out.append("") + out.append(f" assign a0 = {A0_BITS}'({{rd_data_r[{w - 1}:{W1 + W2}], {S0}'b0}});") + out.append(f" assign a1 = {A1_BITS}'($signed({{rd_data_r[{W1 + W2 - 1}:{W2}], {S1}'b0}}));") + out.append(f" assign a2 = {A2_BITS}'($signed({{rd_data_r[{W2 - 1}:0], {S2}'b0}}));") + out.append("") + out.append("endmodule") + out.append("") + return "\n".join(out) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--check", action="store_true") + ap.add_argument("--report", action="store_true") + args = ap.parse_args() + + entries, bases, report = build() + h = emit_h(entries, bases) + sv = emit_sv(entries, bases) + + if args.report: + for name, worst, n, wid in report: + print(f"{name:6s} {n:4d} entries max interpolation error {worst:8.2f} units of 2^-{FRAC}" + f" (2^{math.log2(max(worst, 1e-9) / (1 << FRAC)):.2f}) bits a0/a1/a2 = {wid[0]}/{wid[1]}/{wid[2]}") + print(f"total {len(entries)} entries") + + if args.check: + stale = [] + for path, text in ((OUT_H, h), (OUT_SV, sv)): + try: + with open(path) as fp: + if fp.read() != text: + stale.append(path) + except FileNotFoundError: + stale.append(path) + if stale: + print("stale: " + ", ".join(stale), file=sys.stderr) + return 1 + return 0 + + if not args.report: + os.makedirs(os.path.dirname(OUT_SV), exist_ok=True) + with open(OUT_H, "w") as fp: + fp.write(h) + with open(OUT_SV, "w") as fp: + fp.write(sv) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/syn/extensions.mk b/hw/syn/extensions.mk index 8b549309f5..5d71680331 100644 --- a/hw/syn/extensions.mk +++ b/hw/syn/extensions.mk @@ -90,3 +90,8 @@ ifneq (,$(filter -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_EXT_OM_ENABLE=1 -DVX_CFG_EXT_OM RTL_PKGS += $(RTL_DIR)/om/VX_om_pkg.sv RTL_INCLUDE += -I$(RTL_DIR)/om endif + +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE -DVX_CFG_EXT_MUFU_ENABLE=1 -DVX_CFG_EXT_MUFU_ENABLED=1, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 1bb5edaa98..9a2cde4690 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -10,7 +10,7 @@ # Add a DUT: append its name to DUTS and define the five variables. No new # directory, no dispatcher edit. -DUTS := cache core cp dxa fpu gfx issue lmem mem_unit om raster rtu \ +DUTS := cache core cp dxa fpu gfx issue lmem mem_unit mufu om raster rtu \ scope tcu tensor tex top unittest vm vortex # --------------------------------------------------------------------------- @@ -61,6 +61,13 @@ dxa_PKG = $(RTL_DIR)/dxa/VX_dxa_pkg.sv fpu_IP := 1 fpu_PRJ = $(if $(filter -DVX_CFG_FPU_TYPE_FPNEW,$(XCONFIGS)),VX_fpu_fpnew,$(if $(filter -DVX_CFG_FPU_TYPE_STD,$(XCONFIGS)),VX_fpu_std,VX_fpu_dsp)) fpu_INC = $(BASE_INC) $(FPU_INC) +# BASE_INC pulls in hw/rtl/VX_cluster.sv, which imports VX_tlb_pkg. +fpu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv + +mufu_PRJ := VX_mufu_unit_top +mufu_CFG := -DVX_CFG_EXT_MUFU_ENABLE +mufu_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) -I$(RTL_DIR)/mufu -I$(UNITTEST_DIR)/mufu_unit +mufu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv gfx_PRJ := VX_gfx_top gfx_IP := 1 diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 0387f2a267..001ab90c2a 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -28,6 +28,7 @@ all: $(MAKE) -C fcvt_unit $(MAKE) -C fdivsqrt_unit $(MAKE) -C tex_lerp + $(MAKE) -C mufu_unit run: $(MAKE) -C generic_queue run @@ -59,6 +60,7 @@ run: $(MAKE) -C fcvt_unit run $(MAKE) -C fdivsqrt_unit run $(MAKE) -C tex_lerp run + $(MAKE) -C mufu_unit run # FPU arithmetic units only — executed (not just built) so CI gates correctness. run-fpu: @@ -68,6 +70,10 @@ run-fpu: $(MAKE) -C fcvt_unit run $(MAKE) -C fdivsqrt_unit run +# MUFU datapath, bit-exact against the sw/common golden model. +run-mufu: + $(MAKE) -C mufu_unit run + # TCU FEDP datapath — executed per format so a tcu/tfr change is gated by CI. # Each format must enable its own RTL datapath (FP16-only is the default). Covers # the native TFR and the DPI reference. Float formats compare against the windowed @@ -154,3 +160,4 @@ clean: $(MAKE) -C fcvt_unit clean $(MAKE) -C fdivsqrt_unit clean $(MAKE) -C tex_lerp clean + $(MAKE) -C mufu_unit clean diff --git a/hw/unittest/mufu_unit/Makefile b/hw/unittest/mufu_unit/Makefile new file mode 100644 index 0000000000..cffc5efa2f --- /dev/null +++ b/hw/unittest/mufu_unit/Makefile @@ -0,0 +1,26 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mufu_unit + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +CONFIGS += -DSIMULATION -DVX_CFG_EXT_MUFU_ENABLE + +SRCS += $(SW_COMMON_DIR)/mufu.cpp +SRCS += $(SRC_DIR)/main.cpp + +RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs +RTL_INCLUDE += -I$(RTL_DIR)/mufu + +# Generated headers (VX_config.vh, VX_types.vh) live in the build tree; add them +# as an include-search path only, not a source-discovery root. +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_mufu_pe + +include ../common.mk diff --git a/hw/unittest/mufu_unit/VX_mufu_unit_top.sv b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv new file mode 100644 index 0000000000..0d2e2c0567 --- /dev/null +++ b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv @@ -0,0 +1,37 @@ +`include "VX_define.vh" + +module VX_mufu_unit_top import VX_gpu_pkg::*; ( + input wire clk, + input wire reset, + + input wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_valid, + output wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_ready, + input wire [`VX_CFG_ISSUE_WIDTH-1:0][$bits(dispatch_t)-1:0] dispatch_data, + + output wire [`VX_CFG_ISSUE_WIDTH-1:0] commit_valid, + input wire [`VX_CFG_ISSUE_WIDTH-1:0] commit_ready, + output wire [`VX_CFG_ISSUE_WIDTH-1:0][$bits(commit_t)-1:0] commit_data +); + VX_dispatch_if dispatch_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if [`VX_CFG_ISSUE_WIDTH](); + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_ports + assign dispatch_if[i].valid = dispatch_valid[i]; + assign dispatch_if[i].data = dispatch_data[i]; + assign dispatch_ready[i] = dispatch_if[i].ready; + + assign commit_valid[i] = commit_if[i].valid; + assign commit_data[i] = commit_if[i].data; + assign commit_if[i].ready = commit_ready[i]; + end + + VX_mufu_unit #( + .INSTANCE_ID ("mufu_unit_top") + ) mufu_unit ( + .clk (clk), + .reset (reset), + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + +endmodule diff --git a/hw/unittest/mufu_unit/main.cpp b/hw/unittest/mufu_unit/main.cpp new file mode 100644 index 0000000000..b18596f520 --- /dev/null +++ b/hw/unittest/mufu_unit/main.cpp @@ -0,0 +1,112 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Bit-exact check of the VX_mufu_pe pipeline against the shared golden model +// (sw/common/mufu.cpp) for every function: special values, a strided sweep of +// the whole FP32 space, and random inputs. +// +// usage: mufu_unit [stride] [num_random] + +#include "VVX_mufu_pe.h" +#include "vl_simulator.h" +#include "mufu.h" + +#include +#include +#include +#include +#include +#include + +using Device = VVX_mufu_pe; +using namespace vortex::mufu; + +namespace { + +constexpr int kLatency = 7; + +const uint32_t kSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, + 0x3f800000, 0xbf800000, 0x3f000000, 0x3fc00000, 0x40000000, 0x40800000, + 0x40490fdb, 0x3eaaaaab, 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, + 0x7fc00000, 0xffc00000, 0x7fa00000, 0x42fe0000, 0x43000000, 0xc2fc0000, + 0xc2fe0000, 0xc3000000, 0x4b000001, 0x4b800001, 0x3f7fffff, 0x3f800001, + 0x39800000, 0x39000000, 0x41000000, 0x40ffffff, 0x3a7fffff, 0x3a800000, +}; + +struct Req { uint32_t fn, x; }; + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + + uint64_t stride = 65521; + size_t num_random = 200000; + if (argc > 1) stride = std::strtoull(argv[1], nullptr, 0); + if (argc > 2) num_random = std::strtoull(argv[2], nullptr, 0); + if (stride == 0) stride = 1; + + std::vector reqs; + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { + for (uint32_t x : kSpecials) + reqs.push_back({fn, x}); + for (uint64_t x = 0; x < (1ull << 32); x += stride) + reqs.push_back({fn, uint32_t(x)}); + } + std::mt19937 rng(0x6d756675u); + for (size_t i = 0; i < num_random; ++i) + reqs.push_back({uint32_t(i % FN_COUNT), uint32_t(rng())}); + + vl_simulator sim; + uint64_t ticks = 0; + sim->enable = 1; + sim->fn = 0; + sim->dataa = 0; + ticks = sim.reset(ticks); + + std::deque inflight; + size_t feed = 0, checked = 0; + size_t errors[FN_COUNT] = {}; + size_t total_errors = 0; + + for (size_t c = 0; c < reqs.size() + kLatency; ++c) { + if (feed < reqs.size()) { + sim->fn = reqs[feed].fn; + sim->dataa = reqs[feed].x; + inflight.push_back(reqs[feed]); + ++feed; + } + ticks = sim.step(ticks, 2); + + if (c >= size_t(kLatency - 1) && !inflight.empty()) { + Req r = inflight.front(); + inflight.pop_front(); + uint32_t got = sim->result; + uint32_t exp = eval(r.fn, r.x); + if (got != exp) { + if (total_errors < 20) + printf("MISMATCH %s x=%08x : got %08x expected %08x\n", fn_name(r.fn), r.x, got, exp); + ++errors[r.fn]; + ++total_errors; + } + ++checked; + } + } + + printf("mufu_unit: checked=%zu errors=%zu [", checked, total_errors); + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) + printf("%s%s=%zu", fn ? " " : "", fn_name(fn), errors[fn]); + printf("]\n%s\n", total_errors ? "FAILED" : "PASSED"); + return total_errors ? 1 : 0; +} diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index dfe8ab9bb4..863442440d 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -133,6 +133,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index f4dbfb723a..55d1d9d214 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -130,6 +130,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index f15c7da8a7..ed85464742 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -122,6 +122,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SIM_COMMON_DIR)/dram_sim.cpp $(SIM_COMMON_DIR)/elf_loader.cpp $(SIM_COMMON_DIR)/host_monitor.cpp $(SIM_COMMON_DIR)/cout_drainer.cpp diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 746bd6f7f8..4338c8ab6d 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -55,6 +55,12 @@ ifneq ($(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS)),) SRCS += $(SRC_DIR)/tcu/tcu_tbuf.cpp endif +# Add MUFU extension sources +ifneq ($(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS)),) + SRCS += $(SRC_DIR)/mufu_unit.cpp + SRCS += $(SW_COMMON_DIR)/mufu.cpp +endif + # Add VM sources ifneq ($(filter -DVX_CFG_VM_ENABLE, $(XCONFIGS)),) SRCS += $(SRC_DIR)/mem/mmu.cpp diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index ec729d83e3..b23c4c4dd8 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -47,6 +47,9 @@ #ifdef VX_CFG_EXT_TCU_ENABLE #include "tcu_unit.h" #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +#include "mufu_unit.h" +#endif #ifdef VX_CFG_VM_ENABLE #include "mem/mmu.h" #include @@ -274,6 +277,11 @@ class Core::Impl { #endif #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + snprintf(sname, 100, "%s-mufu", name.c_str()); + func_units_.at((int)FUType::MUFU) = SimPlatform::instance().create_object(sname, simobject_); + #endif + // initialize dispatchers after the units they feed, so a unit takes its // input before the dispatcher fills it in the same cycle. { @@ -286,6 +294,10 @@ class Core::Impl { dispatchers_.at((int)FUType::SFU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_SFU_BLOCKS, VX_CFG_NUM_SFU_LANES, 1); #ifdef VX_CFG_EXT_TCU_ENABLE dispatchers_.at((int)FUType::TCU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_TCU_BLOCKS, VX_CFG_NUM_TCU_LANES, 1); + #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + uint32_t mufu_dly = MufuUnit::kGather ? 2 : 1; + dispatchers_.at((int)FUType::MUFU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_MUFU_BLOCKS, VX_CFG_NUM_MUFU_LANES, mufu_dly); #endif for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { auto& dispatch = dispatchers_.at(fu); @@ -570,6 +582,9 @@ class Core::Impl { case FUType::SFU: ++perf_stats_.sfu_stalls; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: ++perf_stats_.tcu_stalls; break; + #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + case FUType::MUFU: ++perf_stats_.mufu_stalls; break; #endif default: assert(false); } @@ -805,12 +820,15 @@ class Core::Impl { // process completed instructions: one beat per issue slice per cycle, // granted fixed-priority in EX-unit order (ALU highest, then LSU, SFU, - // FPU, TCU) — a busy higher class starves lower ones' writebacks. + // FPU, TCU, MUFU) — a busy higher class starves lower ones' writebacks. static constexpr FUType kCommitPrio[] = { FUType::ALU, FUType::LSU, FUType::SFU, FUType::FPU, #ifdef VX_CFG_EXT_TCU_ENABLE FUType::TCU, #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + FUType::MUFU, + #endif }; for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { SimChannel* granted = nullptr; @@ -868,6 +886,9 @@ class Core::Impl { case FUType::SFU: ++perf_stats_.sfu_instrs; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: ++perf_stats_.tcu_instrs; break; + #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + case FUType::MUFU: ++perf_stats_.mufu_instrs; break; #endif default: assert(false); } diff --git a/sim/simx/core.h b/sim/simx/core.h index df1e117b7b..48936b8fae 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -55,6 +55,9 @@ class Core : public SimObject { uint64_t sfu_stalls = 0; #ifdef VX_CFG_EXT_TCU_ENABLE uint64_t tcu_stalls = 0; + #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + uint64_t mufu_stalls = 0; #endif uint64_t branches = 0; uint64_t divergence = 0; @@ -64,6 +67,9 @@ class Core : public SimObject { uint64_t sfu_instrs = 0; #ifdef VX_CFG_EXT_TCU_ENABLE uint64_t tcu_instrs = 0; + #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + uint64_t mufu_instrs = 0; #endif uint64_t ifetches = 0; uint64_t loads = 0; diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 9369f2cbcb..8bda8d2a36 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -248,6 +248,14 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { } } break; #endif + #ifdef VX_CFG_EXT_MUFU_ENABLE + case VX_DCR_MPM_CLASS_MUFU: { + switch (addr) { + CSR_READ_64(VX_CSR_MPM_MUFU_STALLS, core_perf.mufu_stalls); + CSR_READ_64(VX_CSR_MPM_MUFU_INSTRS, core_perf.mufu_instrs); + } + } break; + #endif #ifdef VX_CFG_EXT_DXA_ENABLE case VX_DCR_MPM_CLASS_DXA: { auto cluster_perf = core_->socket()->cluster()->perf_stats(); diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 9fb835db56..248d89c8b2 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -404,6 +404,13 @@ static op_string_t op_string(const Instr &instr) { return TcuUnit::op_string(tcu_type, tpuArgs); } #endif // VX_CFG_EXT_TCU_ENABLE + #ifdef VX_CFG_EXT_MUFU_ENABLE + ,[&](MufuType mufu_type)-> op_string_t { + std::ostringstream os; + os << mufu_type << ".S"; + return {os.str(), ""}; + } + #endif #ifdef VX_CFG_EXT_TEX_ENABLE ,[&](TexType /*tex_type*/)-> op_string_t { return {"TEX", ""}; @@ -1144,6 +1151,17 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { // known-violation handoff class, checked against its decoded destination. gfx_doctrine::check(*instr); } break; +#ifdef VX_CFG_EXT_MUFU_ENABLE + case Opcode::EXT3: { + if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) + std::abort(); + instr->set_fu_type(FUType::MUFU); + instr->set_op_type(MufuType(rs2)); + instr->set_args(IntrMufuArgs{funct3}); + instr->set_dest_reg(rd, RegType::Float); + instr->set_src_reg(0, rs1, RegType::Float); + } break; +#endif default: std::abort(); } diff --git a/sim/simx/mufu_unit.cpp b/sim/simx/mufu_unit.cpp new file mode 100644 index 0000000000..7ce471867f --- /dev/null +++ b/sim/simx/mufu_unit.cpp @@ -0,0 +1,74 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mufu_unit.h" +#include +#include "debug.h" +#include "core.h" + +using namespace vortex; + +namespace { +// SimX keeps every single-precision FP register value NaN-boxed (see fpu_unit.cpp). +inline uint32_t unbox(uint64_t value) { + if (uint32_t(value >> 32) != 0xffffffff) + return mufu::QNAN_F32; + return uint32_t(value); +} +inline uint64_t box(uint32_t value) { + return value | 0xffffffff00000000; +} +} + +MufuUnit::MufuUnit(const SimContext& ctx, const char* name, Core* core) + // The RTL pipeline has no tag store: what is in flight is bounded by the + // pipeline depth plus its two-entry output buffer. + : FuncUnit(ctx, name, core, VX_CFG_MUFU_LATENCY + 2) +{} + +void MufuUnit::execute(instr_trace_t* trace) { + auto& tmask = trace->tmask; + auto mufu_type = std::get(trace->op_type); + auto& rs1_data = trace->src_data[0]; + trace->dst_data.assign(VX_CFG_NUM_THREADS, reg_data_t{}); + auto& rd_data = trace->dst_data; + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { + if (!tmask.test(t)) + continue; + rd_data[t].u64 = box(mufu::eval(uint32_t(mufu_type), unbox(rs1_data[t].u64))); + } + DT(3, this->name() << " execute: op=" << mufu_type << ", " << *trace); +} + +void MufuUnit::on_tick() { + bool idle = true; + for (uint32_t b = 0; b < VX_CFG_NUM_MUFU_BLOCKS; ++b) { + auto& input = Inputs.at(b); + if (!input.empty()) { + auto& output = Outputs.at(b); + if (!output.full()) { + auto trace = input.peek(); + this->execute(trace); + // Without a lane-gather stage commit reads this port directly, so the + // result also takes the registered stage other units cross into commit. + uint32_t delay = VX_CFG_MUFU_LATENCY + (kGather ? 0 : 1); + output.send(trace, delay); + input.pop(); + } + } + idle &= (input.size() == 0); + } + if (idle) { + this->tick_sleep(); + } +} diff --git a/sim/simx/mufu_unit.h b/sim/simx/mufu_unit.h new file mode 100644 index 0000000000..442daa4629 --- /dev/null +++ b/sim/simx/mufu_unit.h @@ -0,0 +1,36 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "func_unit.h" + +namespace vortex { + +class MufuUnit : public FuncUnit { +public: + MufuUnit(const SimContext& ctx, const char* name, Core*); + + // Partial bandwidth: a lane-gather stage takes results off the response + // port. Otherwise the commit arbiter reads the response port directly. + static constexpr bool kGather = (VX_CFG_NUM_MUFU_BLOCKS != VX_CFG_ISSUE_WIDTH) + || (VX_CFG_NUM_MUFU_LANES != VX_CFG_SIMD_WIDTH); + +protected: + void on_tick() override; + +private: + void execute(instr_trace_t* trace); +}; + +} diff --git a/sim/simx/types.h b/sim/simx/types.h index 1658771c6e..cb20410167 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -191,6 +191,9 @@ enum class FUType { SFU, #ifdef VX_CFG_EXT_TCU_ENABLE TCU, +#endif +#ifdef VX_CFG_EXT_MUFU_ENABLE + MUFU, #endif Count }; @@ -203,6 +206,9 @@ inline std::ostream &operator<<(std::ostream &os, const FUType& type) { case FUType::SFU: os << "SFU"; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: os << "TCU"; break; +#endif +#ifdef VX_CFG_EXT_MUFU_ENABLE + case FUType::MUFU: os << "MUFU"; break; #endif default: assert(false); @@ -592,6 +598,40 @@ inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { /////////////////////////////////////////////////////////////////////////////// +#ifdef VX_CFG_EXT_MUFU_ENABLE +// Values match the instruction's fn field and vortex::mufu::fn_t. +enum class MufuType { + RCP = 0, + RSQRT = 1, + EX2 = 2, + LG2 = 3, + SINPI = 4, + COSPI = 5, + TANH = 6, +}; + +struct IntrMufuArgs { + uint32_t fmt : 3; +}; + +inline std::ostream &operator<<(std::ostream &os, const MufuType& type) { + switch (type) { + case MufuType::RCP: os << "MUFU.RCP"; break; + case MufuType::RSQRT: os << "MUFU.RSQRT"; break; + case MufuType::EX2: os << "MUFU.EX2"; break; + case MufuType::LG2: os << "MUFU.LG2"; break; + case MufuType::SINPI: os << "MUFU.SINPI"; break; + case MufuType::COSPI: os << "MUFU.COSPI"; break; + case MufuType::TANH: os << "MUFU.TANH"; break; + default: + assert(false); + } + return os; +} +#endif + +/////////////////////////////////////////////////////////////////////////////// + enum class WctlType { TMC, WSPAWN, @@ -821,6 +861,9 @@ using OpType = std::variant< #ifdef VX_CFG_EXT_TCU_ENABLE , TcuType #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +, MufuType +#endif #ifdef VX_CFG_EXT_TEX_ENABLE , TexType #endif @@ -848,6 +891,9 @@ using IntrArgs = std::variant< #ifdef VX_CFG_EXT_TCU_ENABLE , IntrTcuArgs #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +, IntrMufuArgs +#endif #ifdef VX_CFG_EXT_TEX_ENABLE , IntrTexArgs #endif diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 4e51883d19..df068a9785 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -130,6 +130,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sw/common/mufu.cpp b/sw/common/mufu.cpp new file mode 100644 index 0000000000..ad5a906620 --- /dev/null +++ b/sw/common/mufu.cpp @@ -0,0 +1,204 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mufu.h" +#include "mufu_tables.h" + +namespace vortex::mufu { + +namespace { + +constexpr uint32_t FRAC = 30; +constexpr uint32_t XL_BITS = 18; + +constexpr uint32_t POS_INF = 0x7f800000; +constexpr uint32_t NEG_INF = 0xff800000; +constexpr uint32_t ONE = 0x3f800000; + +// Output of the range-reduction stage; mirrors the RTL stage-1 register. +struct reduced_t { + bool special; // result is `value`, bypassing interpolation + uint32_t value; + uint32_t addr; // coefficient ROM address + uint32_t xl; // XL_BITS-wide position inside the interval + int32_t exp; // result scale: value = P * 2^-FRAC * 2^exp + bool neg; // result sign + bool zsign; // sign of an exactly-zero result + bool is_lg2; // add `k` as the integer part before normalizing + int32_t k; +}; + +reduced_t reduce(uint32_t fn, uint32_t x) { + reduced_t r{}; + uint32_t s = x >> 31; + uint32_t e = (x >> 23) & 0xff; + uint32_t m = x & 0x7fffff; + bool is_zero = (e == 0); + bool is_inf = (e == 0xff) && (m == 0); + bool is_nan = (e == 0xff) && (m != 0); + uint32_t sig = m | (1u << 23); + + auto special = [&](uint32_t v) { r.special = true; r.value = v; return r; }; + + if (is_nan) + return special(QNAN_F32); + + switch (fn) { + case FN_RCP: + if (is_inf) return special(s << 31); + if (is_zero) return special(s ? NEG_INF : POS_INF); + r.addr = TBL_RCP_BASE + (m >> 16); + r.xl = (m & 0xffff) << 2; + r.exp = 127 - int32_t(e); + r.neg = s; + return r; + + case FN_RSQRT: { + if (is_zero) return special(s ? NEG_INF : POS_INF); + if (s) return special(QNAN_F32); + if (is_inf) return special(0); + int32_t k = int32_t(e) - 127; + r.addr = TBL_RSQRT_BASE + ((uint32_t(k) & 1) << 6) + (m >> 17); + r.xl = (m & 0x1ffff) << 1; + r.exp = -(k >> 1); + return r; + } + + case FN_EX2: { + if (is_inf) return special(s ? 0 : POS_INF); + if (is_zero) return special(ONE); + if (e >= 134) return special(s ? 0 : POS_INF); + int32_t sh = int32_t(e) - 120; + int64_t mag = (sh >= 0) ? (int64_t(sig) << sh) : ((sh > -24) ? int64_t(sig >> -sh) : 0); + int64_t X = s ? -mag : mag; + int64_t f = X & ((int64_t(1) << FRAC) - 1); + r.addr = TBL_EX2_BASE + uint32_t(f >> 24); + r.xl = uint32_t(f >> 6) & ((1u << XL_BITS) - 1); + r.exp = int32_t(X >> FRAC); + return r; + } + + case FN_LG2: + if (is_zero) return special(NEG_INF); + if (s) return special(QNAN_F32); + if (is_inf) return special(POS_INF); + r.addr = TBL_LG2_BASE + (m >> 17); + r.xl = (m & 0x1ffff) << 1; + r.is_lg2 = true; + r.k = int32_t(e) - 127; + return r; + + case FN_SINPI: + case FN_COSPI: { + bool is_sin = (fn == FN_SINPI); + if (is_inf) return special(QNAN_F32); + if (is_zero) return special(is_sin ? (s << 31) : ONE); + // u = 2|x| in quarter turns, 30 fraction bits, modulo 4 + int32_t sh = int32_t(e) - 119; + uint32_t u; + if (sh >= 32) + u = 0; + else if (sh >= 0) + u = uint32_t(uint64_t(sig) << sh); + else + u = (sh > -24) ? (sig >> -sh) : 0; + if (!is_sin) + u += (1u << FRAC); + uint32_t q = u >> FRAC; + uint32_t t = u & ((1u << FRAC) - 1); + r.neg = ((q >> 1) & 1) ^ (is_sin ? s : 0); + r.zsign = is_sin && s; + // Odd quadrants need cos(pi/2 t) = sin(pi/2 (1 - t)); ~t stands in for + // 1 - t, and t == 0 (exactly 1) is returned directly to stay exact. + if (q & 1) { + if (t == 0) + return special(r.neg ? (ONE | 0x80000000) : ONE); + t = ~t & ((1u << FRAC) - 1); + } + r.addr = TBL_SIN_BASE + (t >> 24); + r.xl = (t >> 6) & ((1u << XL_BITS) - 1); + return r; + } + + case FN_TANH: + if (is_inf) return special(s ? (ONE | 0x80000000) : ONE); + if (is_zero) return special(s << 31); + // |x| >= 8 rounds to +-1; below 2^-5, tanh x = x within 2^-11. + if (e >= 130) return special(s ? (ONE | 0x80000000) : ONE); + if (e < 122) return special(x); + // Entries hold 2^-(j+4) * tanh(2^j (1+t)) for binade j = e - 127. + r.addr = TBL_TANH_BASE + ((e - 122) << 3) + (m >> 20); + r.xl = (m >> 2) & ((1u << XL_BITS) - 1); + r.exp = int32_t(e) - 127 + 4; + r.neg = s; + return r; + + default: + return special(0); + } +} + +int64_t interpolate(uint32_t addr, uint32_t xl) { + const coeff_t& c = kTable[addr]; + int64_t a0 = int64_t(c.a0) << S0; + int64_t a1 = int64_t(c.a1) * (int64_t(1) << S1); + int64_t a2 = int64_t(c.a2) * (int64_t(1) << S2); + int64_t sq = (int64_t(xl) * xl) >> XL_BITS; + int64_t t1 = (a1 * xl) >> XL_BITS; + int64_t t2 = (a2 * sq) >> XL_BITS; + int64_t p = a0 + t1 + t2; + return (p < 0) ? 0 : p; +} + +uint32_t normalize(uint64_t mag, int32_t exp, bool neg, bool zsign) { + if (mag == 0) + return uint32_t(zsign) << 31; + int lead = 63 - __builtin_clzll(mag); + uint64_t norm = (mag << (39 - lead)) & ((uint64_t(1) << 40) - 1); + uint32_t sig = uint32_t(norm >> 16); + uint32_t rnd = uint32_t(norm >> 15) & 1; + int32_t bexp = lead - int32_t(FRAC) + exp + 127; + sig += rnd; + if (sig >> 24) { + sig >>= 1; + bexp += 1; + } + uint32_t sgn = uint32_t(neg) << 31; + if (bexp >= 255) + return sgn | POS_INF; + if (bexp <= 0) + return sgn; + return sgn | (uint32_t(bexp) << 23) | (sig & 0x7fffff); +} + +} // namespace + +uint32_t eval(uint32_t fn, uint32_t x) { + reduced_t r = reduce(fn, x); + if (r.special) + return r.value; + int64_t p = interpolate(r.addr, r.xl); + if (r.is_lg2) { + int64_t l = (int64_t(r.k) << FRAC) + p; + bool neg = (l < 0); + return normalize(uint64_t(neg ? -l : l), 0, neg, false); + } + return normalize(uint64_t(p), r.exp, r.neg, r.zsign); +} + +const char* fn_name(uint32_t fn) { + static const char* names[FN_COUNT] = {"rcp", "rsqrt", "ex2", "lg2", "sinpi", "cospi", "tanh"}; + return (fn < FN_COUNT) ? names[fn] : "?"; +} + +} // namespace vortex::mufu diff --git a/sw/common/mufu.h b/sw/common/mufu.h new file mode 100644 index 0000000000..d642b66289 --- /dev/null +++ b/sw/common/mufu.h @@ -0,0 +1,38 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +// Bit-exact model of the MUFU datapath (hw/rtl/mufu). +namespace vortex::mufu { + +enum fn_t : uint32_t { + FN_RCP = 0, + FN_RSQRT = 1, + FN_EX2 = 2, + FN_LG2 = 3, + FN_SINPI = 4, + FN_COSPI = 5, + FN_TANH = 6, + FN_COUNT = 7 +}; + +constexpr uint32_t QNAN_F32 = 0x7fc00000; + +uint32_t eval(uint32_t fn, uint32_t x); + +const char* fn_name(uint32_t fn); + +} // namespace vortex::mufu diff --git a/sw/common/mufu_tables.h b/sw/common/mufu_tables.h new file mode 100644 index 0000000000..31a1276b32 --- /dev/null +++ b/sw/common/mufu_tables.h @@ -0,0 +1,538 @@ +// Generated by hw/scripts/gen_mufu_tables.py. Do not edit. +#pragma once + +#include + +namespace vortex::mufu { + +// Stored coefficients; the datapath uses a0 << S0, a1 << S1, a2 << S2. +struct coeff_t { uint32_t a0; int32_t a1; int32_t a2; }; + +constexpr uint32_t S0 = 2; +constexpr uint32_t S1 = 2; +constexpr uint32_t S2 = 1; + +constexpr uint32_t TBL_RCP_BASE = 0; +constexpr uint32_t TBL_RSQRT_BASE = 128; +constexpr uint32_t TBL_EX2_BASE = 256; +constexpr uint32_t TBL_LG2_BASE = 320; +constexpr uint32_t TBL_SIN_BASE = 384; +constexpr uint32_t TBL_TANH_BASE = 448; +constexpr uint32_t TBL_SIZE = 512; + +constexpr coeff_t kTable[TBL_SIZE] = { + {0x10000000u, -2097098, 32415}, + {0x0fe03f7du, -2064695, 31643}, + {0x0fc0fc0cu, -2033054, 30921}, + {0x0fa232ccu, -2002134, 30221}, + {0x0f83e0f5u, -1971915, 29542}, + {0x0f6603d6u, -1942374, 28883}, + {0x0f4898d3u, -1913493, 28244}, + {0x0f2b9d61u, -1885250, 27623}, + {0x0f0f0f0cu, -1857629, 27020}, + {0x0ef2eb6fu, -1830610, 26435}, + {0x0ed73038u, -1804176, 25867}, + {0x0ebbdb28u, -1778311, 25314}, + {0x0ea0ea0cu, -1752998, 24778}, + {0x0e865ac6u, -1728222, 24256}, + {0x0e6c2b42u, -1703967, 23749}, + {0x0e525980u, -1680219, 23256}, + {0x0e38e38cu, -1656965, 22776}, + {0x0e1fc77eu, -1634189, 22310}, + {0x0e070380u, -1611880, 21856}, + {0x0dee95c2u, -1590025, 21415}, + {0x0dd67c88u, -1568611, 20985}, + {0x0dbeb61cu, -1547627, 20567}, + {0x0da740d9u, -1527062, 20159}, + {0x0d901b1eu, -1506903, 19763}, + {0x0d79435cu, -1487141, 19377}, + {0x0d62b80cu, -1467765, 19000}, + {0x0d4c77aeu, -1448766, 18634}, + {0x0d3680d2u, -1430133, 18277}, + {0x0d20d20cu, -1411857, 17928}, + {0x0d0b69fbu, -1393929, 17589}, + {0x0cf64749u, -1376341, 17258}, + {0x0ce168a6u, -1359083, 16936}, + {0x0cccccccu, -1342148, 16621}, + {0x0cb8727bu, -1325528, 16314}, + {0x0ca4587du, -1309214, 16015}, + {0x0c907da4u, -1293200, 15723}, + {0x0c7ce0c6u, -1277477, 15438}, + {0x0c6980c6u, -1262040, 15160}, + {0x0c565c87u, -1246881, 14888}, + {0x0c4372f7u, -1231993, 14623}, + {0x0c30c30bu, -1217370, 14364}, + {0x0c1e4bbcu, -1203006, 14112}, + {0x0c0c0c0bu, -1188895, 13865}, + {0x0bfa02fdu, -1175031, 13624}, + {0x0be82fa0u, -1161408, 13388}, + {0x0bd69103u, -1148020, 13158}, + {0x0bc52640u, -1134863, 12933}, + {0x0bb3ee71u, -1121930, 12713}, + {0x0ba2e8b9u, -1109217, 12498}, + {0x0b92143fu, -1096720, 12288}, + {0x0b81702du, -1084432, 12083}, + {0x0b70fbb5u, -1072349, 11882}, + {0x0b60b60au, -1060467, 11686}, + {0x0b509e68u, -1048782, 11494}, + {0x0b40b40au, -1037289, 11306}, + {0x0b30f634u, -1025984, 11122}, + {0x0b21642cu, -1014862, 10942}, + {0x0b11fd3au, -1003920, 10766}, + {0x0b02c0b0u, -993155, 10594}, + {0x0af3addcu, -982561, 10425}, + {0x0ae4c416u, -972137, 10260}, + {0x0ad602b5u, -961877, 10098}, + {0x0ac76918u, -951779, 9940}, + {0x0ab8f69eu, -941839, 9785}, + {0x0aaaaaaau, -932053, 9634}, + {0x0a9c84a4u, -922420, 9485}, + {0x0a8e83f5u, -912935, 9340}, + {0x0a80a80au, -903596, 9197}, + {0x0a72f053u, -894399, 9057}, + {0x0a655c43u, -885342, 8920}, + {0x0a57eb50u, -876422, 8786}, + {0x0a4a9cf1u, -867636, 8655}, + {0x0a3d70a3u, -858981, 8526}, + {0x0a3065e4u, -850456, 8400}, + {0x0a237c32u, -842056, 8276}, + {0x0a16b313u, -833781, 8154}, + {0x0a0a0a0au, -825627, 8035}, + {0x09fd809fu, -817591, 7919}, + {0x09f1165eu, -809673, 7804}, + {0x09e4cad2u, -801869, 7692}, + {0x09d89d89u, -794177, 7582}, + {0x09cc8e16u, -786596, 7474}, + {0x09c09c0au, -779123, 7368}, + {0x09b4c6fau, -771755, 7264}, + {0x09a90e7eu, -764492, 7162}, + {0x099d722du, -757330, 7061}, + {0x0991f1a5u, -750269, 6963}, + {0x09868c80u, -743306, 6867}, + {0x097b425fu, -736440, 6772}, + {0x097012e0u, -729668, 6679}, + {0x0964fda6u, -722989, 6588}, + {0x095a0256u, -716402, 6498}, + {0x094f2095u, -709904, 6410}, + {0x09445809u, -703494, 6324}, + {0x0939a85cu, -697170, 6239}, + {0x092f1138u, -690932, 6155}, + {0x09249249u, -684777, 6073}, + {0x091a2b3cu, -678703, 5993}, + {0x090fdbc1u, -672711, 5914}, + {0x0905a386u, -666797, 5836}, + {0x08fb823eu, -660960, 5760}, + {0x08f1779eu, -655201, 5685}, + {0x08e78357u, -649516, 5611}, + {0x08dda520u, -643904, 5539}, + {0x08d3dcb0u, -638365, 5468}, + {0x08ca29c0u, -632898, 5398}, + {0x08c08c09u, -627500, 5329}, + {0x08b70344u, -622171, 5262}, + {0x08ad8f30u, -616910, 5195}, + {0x08a42f87u, -611715, 5130}, + {0x089ae408u, -606585, 5065}, + {0x0891ac74u, -601520, 5002}, + {0x08888888u, -596517, 4940}, + {0x087f7808u, -591577, 4879}, + {0x08767ab6u, -586698, 4819}, + {0x086d9054u, -581880, 4760}, + {0x0864b8a8u, -577120, 4702}, + {0x085bf376u, -572418, 4644}, + {0x08534085u, -567774, 4588}, + {0x084a9f9cu, -563186, 4533}, + {0x08421084u, -558654, 4478}, + {0x08399305u, -554175, 4425}, + {0x083126eau, -549751, 4372}, + {0x0828cbfcu, -545379, 4320}, + {0x08208208u, -541059, 4269}, + {0x081848dbu, -536791, 4218}, + {0x08102040u, -532572, 4169}, + {0x08080808u, -528404, 4120}, + {0x10000000u, -2097017, 48275}, + {0x0fe05ebbu, -2048776, 46389}, + {0x0fc1763cu, -2002395, 44664}, + {0x0fa33f8cu, -1957737, 43028}, + {0x0f85b41du, -1914715, 41475}, + {0x0f68cdb4u, -1873246, 39999}, + {0x0f4c8666u, -1833252, 38596}, + {0x0f30d896u, -1794662, 37260}, + {0x0f15bee9u, -1757407, 35989}, + {0x0efb3448u, -1721422, 34777}, + {0x0ee133dau, -1686650, 33622}, + {0x0ec7b8ffu, -1653032, 32519}, + {0x0eaebf50u, -1620517, 31467}, + {0x0e964295u, -1589053, 30462}, + {0x0e7e3ecdu, -1558595, 29501}, + {0x0e66b01fu, -1529098, 28582}, + {0x0e4f92dfu, -1500519, 27702}, + {0x0e38e38au, -1472819, 26860}, + {0x0e229ec5u, -1445962, 26054}, + {0x0e0cc155u, -1419910, 25281}, + {0x0df74828u, -1394632, 24539}, + {0x0de23046u, -1370095, 23828}, + {0x0dcd76dau, -1346269, 23145}, + {0x0db9192bu, -1323126, 22490}, + {0x0da5149bu, -1300638, 21860}, + {0x0d9166a8u, -1278780, 21254}, + {0x0d7e0ceau, -1257528, 20672}, + {0x0d6b050cu, -1236857, 20112}, + {0x0d584cd4u, -1216747, 19573}, + {0x0d45e21eu, -1197176, 19053}, + {0x0d33c2d8u, -1178124, 18553}, + {0x0d21ed03u, -1159572, 18071}, + {0x0d105eb6u, -1141502, 17607}, + {0x0cff1616u, -1123896, 17159}, + {0x0cee115du, -1106739, 16727}, + {0x0cdd4ed1u, -1090013, 16310}, + {0x0ccccccbu, -1073705, 15907}, + {0x0cbc89b0u, -1057799, 15518}, + {0x0cac83f4u, -1042282, 15142}, + {0x0c9cba1au, -1027140, 14779}, + {0x0c8d2aafu, -1012362, 14428}, + {0x0c7dd44fu, -997935, 14089}, + {0x0c6eb5a1u, -983847, 13760}, + {0x0c5fcd57u, -970088, 13442}, + {0x0c511a2du, -956646, 13135}, + {0x0c429aebu, -943512, 12837}, + {0x0c344e63u, -930676, 12549}, + {0x0c26336eu, -918128, 12269}, + {0x0c1848f2u, -905859, 11998}, + {0x0c0a8ddbu, -893862, 11736}, + {0x0bfd011fu, -882127, 11481}, + {0x0befa1bau, -870646, 11234}, + {0x0be26eb2u, -859413, 10995}, + {0x0bd56714u, -848418, 10762}, + {0x0bc889f5u, -837657, 10537}, + {0x0bbbd66eu, -827120, 10318}, + {0x0baf4ba2u, -816803, 10105}, + {0x0ba2e8bau, -806699, 9898}, + {0x0b96ace2u, -796801, 9697}, + {0x0b8a974fu, -787104, 9502}, + {0x0b7ea73cu, -777602, 9313}, + {0x0b72dbe8u, -768290, 9128}, + {0x0b673498u, -759162, 8949}, + {0x0b5bb097u, -750214, 8774}, + {0x0b504f2du, -1482786, 34088}, + {0x0b39f191u, -1448703, 32802}, + {0x0b241696u, -1415907, 31582}, + {0x0b0eb956u, -1384329, 30426}, + {0x0af9d52eu, -1353908, 29327}, + {0x0ae565b7u, -1324585, 28284}, + {0x0ad166c1u, -1296305, 27291}, + {0x0abdd454u, -1269018, 26347}, + {0x0aaaaaa6u, -1242674, 25448}, + {0x0a97e620u, -1217230, 24591}, + {0x0a858351u, -1192642, 23774}, + {0x0a737ef6u, -1168870, 22995}, + {0x0a61d5f1u, -1145878, 22250}, + {0x0a508548u, -1123630, 21540}, + {0x0a3f8a25u, -1102093, 20860}, + {0x0a2ee1cfu, -1081235, 20210}, + {0x0a1e89aeu, -1061027, 19589}, + {0x0a0e7f48u, -1041441, 18993}, + {0x09fec039u, -1022449, 18423}, + {0x09ef4a3eu, -1004028, 17876}, + {0x09e01b26u, -986154, 17352}, + {0x09d130dau, -968803, 16849}, + {0x09c2895bu, -951956, 16366}, + {0x09b422b9u, -935591, 15903}, + {0x09a5fb1cu, -919690, 15457}, + {0x099810beu, -904234, 15029}, + {0x098a61eau, -889206, 14617}, + {0x097cecfdu, -874590, 14221}, + {0x096fb061u, -860370, 13840}, + {0x0962aa93u, -846531, 13473}, + {0x0955da1du, -833060, 13119}, + {0x09493d95u, -819941, 12778}, + {0x093cd3a2u, -807164, 12450}, + {0x09309af3u, -794715, 12133}, + {0x09249248u, -782582, 11828}, + {0x0918b868u, -770756, 11533}, + {0x090d0c28u, -759224, 11248}, + {0x09018c65u, -747977, 10973}, + {0x08f63808u, -737004, 10707}, + {0x08eb0e02u, -726298, 10450}, + {0x08e00d4fu, -715848, 10202}, + {0x08d534f2u, -705647, 9962}, + {0x08ca83f6u, -695685, 9730}, + {0x08bff96fu, -685956, 9505}, + {0x08b5947au, -676451, 9288}, + {0x08ab5439u, -667164, 9077}, + {0x08a137d5u, -658087, 8873}, + {0x08973e80u, -649214, 8676}, + {0x088d6772u, -640539, 8484}, + {0x0883b1e7u, -632056, 8298}, + {0x087a1d22u, -623758, 8118}, + {0x0870a86eu, -615640, 7944}, + {0x08675318u, -607696, 7774}, + {0x085e1c75u, -599922, 7610}, + {0x085503deu, -592313, 7451}, + {0x084c08b1u, -584862, 7296}, + {0x08432a51u, -577567, 7145}, + {0x083a6824u, -570422, 6999}, + {0x0831c199u, -563424, 6857}, + {0x0829361cu, -556567, 6719}, + {0x0820c524u, -549848, 6585}, + {0x08186e27u, -543263, 6455}, + {0x081030a2u, -536809, 6328}, + {0x08080c14u, -530481, 6204}, + {0x10000000u, 2907245, 31646}, + {0x102c9a46u, 2938896, 32003}, + {0x1059b0dbu, 2970898, 32351}, + {0x10874520u, 3003249, 32704}, + {0x10b55875u, 3035952, 33060}, + {0x10e3ec3bu, 3069012, 33420}, + {0x111301d8u, 3102431, 33784}, + {0x11429ab7u, 3136214, 34151}, + {0x1172b845u, 3170365, 34523}, + {0x11a35bf4u, 3204888, 34899}, + {0x11d4873au, 3239787, 35279}, + {0x12063b91u, 3275066, 35663}, + {0x12387a77u, 3310729, 36052}, + {0x126b456fu, 3346781, 36444}, + {0x129e9dfeu, 3383225, 36841}, + {0x12d285b0u, 3420066, 37242}, + {0x1306fe13u, 3457308, 37648}, + {0x133c08bbu, 3494956, 38058}, + {0x1371a740u, 3533013, 38472}, + {0x13a7db3eu, 3571485, 38891}, + {0x13dea656u, 3610376, 39315}, + {0x14160a2cu, 3649690, 39743}, + {0x144e086au, 3689433, 40176}, + {0x1486a2c0u, 3729608, 40613}, + {0x14bfdadfu, 3770221, 41055}, + {0x14f9b281u, 3811276, 41502}, + {0x15342b61u, 3852778, 41954}, + {0x156f4741u, 3894732, 42411}, + {0x15ab07e8u, 3937143, 42873}, + {0x15e76f20u, 3980016, 43340}, + {0x16247ebbu, 4023355, 43812}, + {0x1662388du, 4067167, 44289}, + {0x16a09e71u, 4111455, 44771}, + {0x16dfb247u, 4156226, 45259}, + {0x171f75f4u, 4201484, 45752}, + {0x175feb61u, 4247236, 46250}, + {0x17a1147fu, 4293485, 46753}, + {0x17e2f342u, 4340238, 47263}, + {0x182589a5u, 4387500, 47777}, + {0x1868d9a7u, 4435277, 48297}, + {0x18ace54eu, 4483574, 48823}, + {0x18f1aea5u, 4532397, 49355}, + {0x193737bdu, 4581751, 49892}, + {0x197d82acu, 4631643, 50436}, + {0x19c4918fu, 4682078, 50985}, + {0x1a0c6688u, 4733063, 51540}, + {0x1a5503beu, 4784603, 52101}, + {0x1a9e6b62u, 4836704, 52669}, + {0x1ae89fa6u, 4889372, 53242}, + {0x1b33a2c5u, 4942614, 53822}, + {0x1b7f7700u, 4996435, 54408}, + {0x1bcc1e9du, 5050843, 55001}, + {0x1c199bebu, 5105843, 55599}, + {0x1c67f13cu, 5161442, 56205}, + {0x1cb720eau, 5217646, 56817}, + {0x1d072d58u, 5274463, 57436}, + {0x1d5818ebu, 5331898, 58061}, + {0x1da9e612u, 5389958, 58693}, + {0x1dfc9742u, 5448651, 59332}, + {0x1e502ef6u, 5507983, 59979}, + {0x1ea4afb1u, 5567961, 60632}, + {0x1efa1bfdu, 5628592, 61292}, + {0x1f50766au, 5689883, 61959}, + {0x1fa7c190u, 5751842, 62634}, + {0x00000000u, 6050894, -93196}, + {0x005b9e69u, 5957748, -90271}, + {0x00b5d6aau, 5867487, -87576}, + {0x010eb397u, 5779920, -85001}, + {0x01663f7cu, 5694928, -82537}, + {0x01bc8430u, 5612399, -80179}, + {0x02118b1du, 5532228, -77920}, + {0x02655d48u, 5454315, -75756}, + {0x02b80352u, 5378566, -73680}, + {0x03098584u, 5304892, -71689}, + {0x0359ebd0u, 5233209, -69777}, + {0x03a93dd3u, 5163437, -67941}, + {0x03f782e0u, 5095502, -66176}, + {0x0444c200u, 5029330, -64480}, + {0x049101f3u, 4964856, -62847}, + {0x04dc493cu, 4902013, -61276}, + {0x05269e1bu, 4840742, -59763}, + {0x05700696u, 4780983, -58305}, + {0x05b8887bu, 4722681, -56901}, + {0x06002960u, 4665784, -55546}, + {0x0646eeaau, 4610242, -54239}, + {0x068cdd8au, 4556007, -52978}, + {0x06d1fb04u, 4503033, -51760}, + {0x07164bf2u, 4451276, -50583}, + {0x0759d4feu, 4400696, -49447}, + {0x079c9aaeu, 4351252, -48348}, + {0x07dea160u, 4302907, -47285}, + {0x081fed4bu, 4255625, -46257}, + {0x08608286u, 4209370, -45263}, + {0x08a06502u, 4164110, -44300}, + {0x08df9894u, 4119813, -43367}, + {0x091e20efu, 4076448, -42463}, + {0x095c01a8u, 4033987, -41588}, + {0x09993e3au, 3992401, -40739}, + {0x09d5da02u, 3951664, -39916}, + {0x0a11d844u, 3911749, -39118}, + {0x0a4d3c2au, 3872633, -38343}, + {0x0a8808c8u, 3834292, -37591}, + {0x0ac24118u, 3796702, -36862}, + {0x0afbe7feu, 3759842, -36153}, + {0x0b35004bu, 3723691, -35464}, + {0x0b6d8cb9u, 3688228, -34795}, + {0x0ba58fefu, 3653435, -34145}, + {0x0bdd0c80u, 3619292, -33512}, + {0x0c1404eeu, 3585781, -32897}, + {0x0c4a7ba9u, 3552885, -32299}, + {0x0c80730eu, 3520587, -31717}, + {0x0cb5ed6cu, 3488871, -31151}, + {0x0ceaed02u, 3457721, -30600}, + {0x0d1f73fdu, 3427123, -30063}, + {0x0d53847eu, 3397061, -29540}, + {0x0d872096u, 3367522, -29031}, + {0x0dba4a4au, 3338493, -28534}, + {0x0ded0392u, 3309959, -28051}, + {0x0e1f4e54u, 3281910, -27579}, + {0x0e512c71u, 3254331, -27120}, + {0x0e829fb9u, 3227213, -26671}, + {0x0eb3a9f2u, 3200542, -26234}, + {0x0ee44cd8u, 3174309, -25808}, + {0x0f148a19u, 3148502, -25391}, + {0x0f44635cu, 3123111, -24985}, + {0x0f73da3bu, 3098127, -24588}, + {0x0fa2f048u, 3073539, -24201}, + {0x0fd1a70bu, 3049339, -23823}, + {0x00000000u, 6588682, -1841}, + {0x00648550u, 6586785, -5952}, + {0x00c8fb28u, 6580833, -9916}, + {0x012d5201u, 6570917, -13874}, + {0x01917a64u, 6557043, -17823}, + {0x01f564ddu, 6539219, -21762}, + {0x02590206u, 6517456, -25688}, + {0x02bc4280u, 6491767, -29598}, + {0x031f16ffu, 6462168, -33490}, + {0x03817045u, 6428676, -37362}, + {0x03e33f27u, 6391312, -41212}, + {0x04447491u, 6350097, -45037}, + {0x04a50184u, 6305058, -48835}, + {0x0504d71eu, 6256221, -52603}, + {0x0563e696u, 6203616, -56340}, + {0x05c22145u, 6147273, -60042}, + {0x061f78a2u, 6087228, -63709}, + {0x067bde4au, 6023516, -67337}, + {0x06d743fbu, 5956176, -70925}, + {0x07319ba0u, 5885247, -74470}, + {0x078ad747u, 5810774, -77970}, + {0x07e2e930u, 5732801, -81423}, + {0x0839c3c6u, 5651374, -84827}, + {0x088f59a4u, 5566543, -88180}, + {0x08e39d96u, 5478360, -91479}, + {0x093682a0u, 5386876, -94724}, + {0x0987fbf8u, 5292147, -97912}, + {0x09d7fd0eu, 5194231, -101040}, + {0x0a26798cu, 5093185, -104108}, + {0x0a736558u, 4989072, -107113}, + {0x0abeb494u, 4881954, -110054}, + {0x0b085ba4u, 4771895, -112928}, + {0x0b504f2eu, 4658961, -115734}, + {0x0b968416u, 4543221, -118471}, + {0x0bdaef8cu, 4424744, -121136}, + {0x0c1d8700u, 4303603, -123728}, + {0x0c5e4031u, 4179868, -126246}, + {0x0c9d1120u, 4053616, -128688}, + {0x0cd9f020u, 3924922, -131052}, + {0x0d14d3ccu, 3793864, -133337}, + {0x0d4db310u, 3660521, -135542}, + {0x0d848528u, 3524973, -137665}, + {0x0db9419eu, 3387301, -139705}, + {0x0debe052u, 3247589, -141661}, + {0x0e1c5975u, 3105921, -143532}, + {0x0e4aa58du, 2962382, -145317}, + {0x0e76bd76u, 2817059, -147013}, + {0x0ea09a66u, 2670038, -148622}, + {0x0ec835e5u, 2521409, -150140}, + {0x0eed89d9u, 2371262, -151569}, + {0x0f109080u, 2219686, -152906}, + {0x0f314474u, 2066773, -154151}, + {0x0f4fa0a9u, 1912615, -155303}, + {0x0f6ba072u, 1757305, -156361}, + {0x0f853f7cu, 1600937, -157325}, + {0x0f9c79d4u, 1443604, -158195}, + {0x0fb14be7u, 1285401, -158969}, + {0x0fc3b27cu, 1126425, -159648}, + {0x0fd3aabeu, 966770, -160230}, + {0x0fe13238u, 806532, -160716}, + {0x0fec46d1u, 645809, -161105}, + {0x0ff4e6d6u, 484696, -161397}, + {0x0ffb10f1u, 323292, -161591}, + {0x0ffec430u, 161693, -161689}, + {0x00ffeab1u, 2095111, -543}, + {0x011fe1a8u, 2094568, -607}, + {0x013fd660u, 2093961, -670}, + {0x015fc899u, 2093291, -734}, + {0x017fb814u, 2092557, -797}, + {0x019fa492u, 2091759, -861}, + {0x01bf8dd2u, 2090898, -924}, + {0x01df7396u, 2089974, -987}, + {0x00ffaad0u, 2089005, -2163}, + {0x011f86c0u, 2086842, -2414}, + {0x013f59c0u, 2084427, -2664}, + {0x015f22d5u, 2081763, -2912}, + {0x017ee105u, 2078851, -3160}, + {0x019e9359u, 2075691, -3405}, + {0x01be38dbu, 2072286, -3649}, + {0x01ddd096u, 2068636, -3891}, + {0x00feacc8u, 2064812, -8502}, + {0x011e1dcfu, 2056309, -9447}, + {0x013d6bc7u, 2046860, -10374}, + {0x015c9307u, 2036484, -11282}, + {0x017b8ff8u, 2025201, -12169}, + {0x019a5f1bu, 2013030, -13033}, + {0x01b8fd04u, 1999995, -13874}, + {0x01d7665cu, 1986119, -14689}, + {0x00facbe9u, 1971642, -31722}, + {0x0118a38fu, 1939902, -34656}, + {0x0135f980u, 1905227, -37353}, + {0x0152c2bdu, 1867854, -39803}, + {0x016ef537u, 1828031, -41997}, + {0x018a87dcu, 1786014, -43931}, + {0x01a5729au, 1742062, -45605}, + {0x01bfae68u, 1696437, -47018}, + {0x00ec9a8fu, 1649657, -97285}, + {0x01050869u, 1552225, -99966}, + {0x011bf481u, 1452129, -100861}, + {0x013157eau, 1351155, -100166}, + {0x0145324fu, 1250893, -98128}, + {0x01578914u, 1152689, -94986}, + {0x0168666au, 1057644, -90980}, + {0x0177d854u, 966623, -86335}, + {0x00c2f848u, 878759, -157188}, + {0x00cf2eceu, 721543, -135503}, + {0x00d9298au, 586134, -114288}, + {0x00e13cb5u, 472007, -94752}, + {0x00e7b821u, 377446, -77498}, + {0x00ece3c5u, 300142, -62714}, + {0x00f0fe38u, 237609, -50328}, + {0x00f43c80u, 187441, -40124}, + {0x007b65ceu, 145562, -56829}, + {0x007d304fu, 89495, -35261}, + {0x007e4998u, 54733, -21685}, + {0x007ef565u, 33365, -13263}, + {0x007f5e0au, 20299, -8086}, + {0x007f9dacu, 12335, -4920}, + {0x007fc453u, 7490, -2989}, + {0x007fdbcbu, 4546, -1815}, + {0x003ff50cu, 2635, -1750}, + {0x003ffbf8u, 970, -644}, + {0x003ffe84u, 357, -237}, + {0x003fff75u, 131, -87}, + {0x003fffcdu, 48, -32}, + {0x003fffedu, 18, -12}, + {0x003ffff9u, 7, -4}, + {0x003ffffeu, 2, -2}, +}; + +} // namespace vortex::mufu diff --git a/sw/kernel/include/vx_mufu.h b/sw/kernel/include/vx_mufu.h new file mode 100644 index 0000000000..5ce4480b81 --- /dev/null +++ b/sw/kernel/include/vx_mufu.h @@ -0,0 +1,106 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// MUFU special-function instructions (custom-2, requires VX_CFG_EXT_MUFU_ENABLE). +// Results are approximate: subnormals +// flush to zero, frm is ignored and fflags are never raised. +// +// Encoding: .insn r CUSTOM2, fmt=funct3, 0=funct7, fd, fs1, x +// The function select rides the rs2 field, so it is written as a literal xN. + +#ifndef __VX_MUFU_H__ +#define __VX_MUFU_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define __VX_MUFU_S(name, fn) \ +__attribute__((always_inline)) \ +inline float name(float x) { \ + float r; \ + __asm__ (".insn r %1, 0, 0, %0, %2, x" #fn \ + : "=f"(r) : "i"(RISCV_CUSTOM2), "f"(x)); \ + return r; \ +} + +__VX_MUFU_S(vx_rcpf, 0) // 1/x +__VX_MUFU_S(vx_rsqrtf, 1) // 1/sqrt(x) +__VX_MUFU_S(vx_ex2f, 2) // 2^x +__VX_MUFU_S(vx_lg2f, 3) // log2(x) +__VX_MUFU_S(vx_sinpif, 4) // sin(pi*x) +__VX_MUFU_S(vx_cospif, 5) // cos(pi*x) +__VX_MUFU_S(vx_tanhf, 6) // tanh(x) + +#undef __VX_MUFU_S + +// Fast-math compositions, as CUDA's __expf/__logf/__sinf/__cosf build on MUFU. + +__attribute__((always_inline)) +inline float vx_expf_fast(float x) { + return vx_ex2f(x * 1.44269504088896341f); +} + +__attribute__((always_inline)) +inline float vx_logf_fast(float x) { + return vx_lg2f(x) * 0.693147180559945309f; +} + +__attribute__((always_inline)) +inline float vx_sinf_fast(float x) { + return vx_sinpif(x * 0.318309886183790672f); +} + +__attribute__((always_inline)) +inline float vx_cosf_fast(float x) { + return vx_cospif(x * 0.318309886183790672f); +} + +__attribute__((always_inline)) +inline float vx_sqrtf_fast(float x) { + // x * rsqrt(x) is 0 * inf = NaN at both ends. + float r = x * vx_rsqrtf(x); + return (x == 0.0f || x == __builtin_inff()) ? x : r; +} + +__attribute__((always_inline)) +inline float vx_divf_fast(float a, float b) { + return a * vx_rcpf(b); +} + +// Activations. + +__attribute__((always_inline)) +inline float vx_sigmoidf_fast(float x) { + return 0.5f + 0.5f * vx_tanhf(0.5f * x); +} + +__attribute__((always_inline)) +inline float vx_siluf_fast(float x) { + return x * vx_sigmoidf_fast(x); +} + +// GELU, tanh form: 0.5x(1 + tanh(sqrt(2/pi)(x + 0.044715x^3))). +__attribute__((always_inline)) +inline float vx_geluf_fast(float x) { + float u = 0.797884560802865356f * x * (1.0f + 0.044715f * x * x); + return 0.5f * x * (1.0f + vx_tanhf(u)); +} + +#ifdef __cplusplus +} +#endif + +#endif // __VX_MUFU_H__ diff --git a/sw/runtime/common/perf.cpp b/sw/runtime/common/perf.cpp index 4abbdc869e..f716430498 100644 --- a/sw/runtime/common/perf.cpp +++ b/sw/runtime/common/perf.cpp @@ -273,6 +273,7 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { const bool fpu_en = (isa_flags & VX_ISA_STD_F) != 0; const bool tcu_en = (isa_flags & VX_ISA_EXT_TCU) != 0; + const bool mufu_en = (isa_flags & VX_ISA_EXT_MUFU) != 0; const bool tex_en = (isa_flags & VX_ISA_EXT_TEX) != 0; const bool raster_en = (isa_flags & VX_ISA_EXT_RASTER) != 0; const bool om_en = (isa_flags & VX_ISA_EXT_OM) != 0; @@ -656,6 +657,23 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { tot_tbuf_stalls, tot_tbuf_cache_hits, tot_lmem_reads); } break; + case VX_DCR_MPM_CLASS_MUFU: { + if (!mufu_en) { + perf_print(stream, "MUFU not supported"); + break; + } + uint64_t tot_stalls = 0, tot_instrs = 0; + for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { + uint64_t stalls = 0, instrs = 0; + CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_MUFU_STALLS, core_id, &stalls), { return err; }); + CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_MUFU_INSTRS, core_id, &instrs), { return err; }); + perf_print_core(stream, core_id, "mufu: instrs=%" PRIu64 ", dispatch_stalls=%" PRIu64, instrs, stalls); + tot_stalls += stalls; + tot_instrs += instrs; + } + perf_print(stream, "mufu: total_instrs=%" PRIu64 ", total_dispatch_stalls=%" PRIu64, tot_instrs, tot_stalls); + } break; + case VX_DCR_MPM_CLASS_TEX: { if (!tex_en) { perf_print(stream, "TEX not supported"); diff --git a/sw/runtime/include/vortex2.h b/sw/runtime/include/vortex2.h index 0f79760b9c..7e2e99d6bb 100644 --- a/sw/runtime/include/vortex2.h +++ b/sw/runtime/include/vortex2.h @@ -109,6 +109,7 @@ typedef struct vx_kernel* vx_kernel_h; #define VX_ISA_EXT_TCU (1ull << (32 + 9)) #define VX_ISA_EXT_DXA (1ull << (32 + 10)) #define VX_ISA_EXT_RTU (1ull << (32 + 11)) +#define VX_ISA_EXT_MUFU (1ull << (32 + 12)) // ============================================================================ // Device memory access flags (vx_buffer_create / vx_buffer_access) diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e50ebfff69..21d7960bfe 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax \ + softmax mufu \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/mufu/Makefile b/tests/regression/mufu/Makefile new file mode 100644 index 0000000000..cdd955d62e --- /dev/null +++ b/tests/regression/mufu/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_MUFU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_MUFU_ENABLE) + +PROJECT := mufu + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp $(SW_COMMON_DIR)/mufu.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +VX_HDRS := $(VORTEX_HOME)/sw/kernel/include/vx_mufu.h $(SRC_DIR)/common.h + +OPTS ?= -n4096 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/mufu/common.h b/tests/regression/mufu/common.h new file mode 100644 index 0000000000..18930ab03c --- /dev/null +++ b/tests/regression/mufu/common.h @@ -0,0 +1,36 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Ops 0..6 are the raw MUFU functions (same numbering as the instruction's fn +// field); the rest are the fast-math compositions from vx_mufu.h. +enum { + OP_RCP = 0, + OP_RSQRT, + OP_EX2, + OP_LG2, + OP_SINPI, + OP_COSPI, + OP_TANH, + OP_EXPF, + OP_LOGF, + OP_SINF, + OP_COSF, + OP_SQRTF, + OP_SIGMOID, + OP_SILU, + OP_GELU, + OP_COUNT +}; + +typedef struct { + uint32_t op; + uint32_t num_points; + uint32_t repeat; + uint32_t padding; + uint64_t src_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/mufu/kernel.cpp b/tests/regression/mufu/kernel.cpp new file mode 100644 index 0000000000..fb19b4067a --- /dev/null +++ b/tests/regression/mufu/kernel.cpp @@ -0,0 +1,41 @@ +#include +#include +#include "common.h" + +template +static void apply(const float* src, float* dst, uint32_t n, uint32_t repeat, F f) { + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) + return; + float y = f(src[i]); + // Extra rounds chain each result into the next call; they lengthen runs for + // the SimX/RTL cycle-parity check while keeping the result reproducible. + for (uint32_t r = 1; r < repeat; ++r) + y = f(y); + dst[i] = y; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + uint32_t n = arg->num_points; + uint32_t k = arg->repeat; + switch (arg->op) { + case OP_RCP: apply(src, dst, n, k, vx_rcpf); break; + case OP_RSQRT: apply(src, dst, n, k, vx_rsqrtf); break; + case OP_EX2: apply(src, dst, n, k, vx_ex2f); break; + case OP_LG2: apply(src, dst, n, k, vx_lg2f); break; + case OP_SINPI: apply(src, dst, n, k, vx_sinpif); break; + case OP_COSPI: apply(src, dst, n, k, vx_cospif); break; + case OP_TANH: apply(src, dst, n, k, vx_tanhf); break; + case OP_EXPF: apply(src, dst, n, k, vx_expf_fast); break; + case OP_LOGF: apply(src, dst, n, k, vx_logf_fast); break; + case OP_SINF: apply(src, dst, n, k, vx_sinf_fast); break; + case OP_COSF: apply(src, dst, n, k, vx_cosf_fast); break; + case OP_SQRTF: apply(src, dst, n, k, vx_sqrtf_fast); break; + case OP_SIGMOID: apply(src, dst, n, k, vx_sigmoidf_fast); break; + case OP_SILU: apply(src, dst, n, k, vx_siluf_fast); break; + case OP_GELU: apply(src, dst, n, k, vx_geluf_fast); break; + default: break; + } +} diff --git a/tests/regression/mufu/main.cpp b/tests/regression/mufu/main.cpp new file mode 100644 index 0000000000..70aa31bf7b --- /dev/null +++ b/tests/regression/mufu/main.cpp @@ -0,0 +1,302 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +static const char* kOpNames[OP_COUNT] = { + "rcp", "rsqrt", "ex2", "lg2", "sinpi", "cospi", "tanh", + "expf", "logf", "sinf", "cosf", "sqrtf", "sigmoid", "silu", "gelu" +}; + +static const uint32_t kSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, + 0x3f800000, 0xbf800000, 0x3f000000, 0x40000000, 0x40800000, 0x40490fdb, + 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, 0x7fc00000, 0x7fa00000, + 0x42fe0000, 0x43000000, 0xc2fc0000, 0xc3000000, 0x4b000001, 0x3f7fffff, + 0x3f800001, 0x39800000, 0x41000000, 0x40ffffff, +}; + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_points = 4096; +uint32_t repeat = 1; +int selected_op = -1; + +vx_device_h device = nullptr; +vx_buffer_h src_buffer = nullptr; +vx_buffer_h dst_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; + +static void show_usage() { + std::cout << "Vortex MUFU Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n points] [-r repeat] [-t op] [-h: help]" << std::endl; + std::cout << " ops:"; + for (auto name : kOpNames) + std::cout << " " << name; + std::cout << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:r:t:k:h")) != -1) { + switch (c) { + case 'n': + num_points = atoi(optarg); + break; + case 'r': + repeat = std::max(1, atoi(optarg)); + break; + case 't': + for (int i = 0; i < OP_COUNT; ++i) { + if (strcmp(optarg, kOpNames[i]) == 0) + selected_op = i; + } + if (selected_op < 0) { + show_usage(); + exit(-1); + } + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (src_buffer) vx_buffer_release(src_buffer); + if (dst_buffer) vx_buffer_release(dst_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +static float as_float(uint32_t u) { float f; memcpy(&f, &u, 4); return f; } +static uint32_t as_bits(float f) { uint32_t u; memcpy(&u, &f, 4); return u; } + +static float rand_range(std::mt19937& rng, float lo, float hi) { + return std::uniform_real_distribution(lo, hi)(rng); +} + +// Log-uniform magnitude in [2^lo, 2^hi], random sign when `sign`. +static float rand_log(std::mt19937& rng, float lo, float hi, bool sign) { + float m = std::exp2(rand_range(rng, lo, hi)); + return (sign && (rng() & 1)) ? -m : m; +} + +static float gen_input(int op, std::mt19937& rng) { + // A quarter of raw-op inputs are arbitrary bit patterns. + if (op <= OP_TANH && (rng() & 3) == 0) + return as_float(uint32_t(rng())); + switch (op) { + case OP_RCP: return rand_log(rng, -125, 125, true); + case OP_RSQRT: return rand_log(rng, -125, 125, false); + case OP_EX2: return rand_range(rng, -130.0f, 130.0f); + case OP_LG2: return (rng() & 1) ? rand_range(rng, 0.5f, 2.0f) : rand_log(rng, -125, 125, false); + case OP_SINPI: + case OP_COSPI: return (rng() & 1) ? rand_range(rng, -2.0f, 2.0f) : rand_range(rng, -4096.0f, 4096.0f); + case OP_TANH: return (rng() & 1) ? rand_range(rng, -10.0f, 10.0f) : rand_log(rng, -20, 3, true); + case OP_EXPF: return rand_range(rng, -80.0f, 80.0f); + case OP_LOGF: return rand_log(rng, -120, 120, false); + case OP_SINF: + case OP_COSF: return rand_range(rng, -100.0f, 100.0f); + case OP_SQRTF: return rand_log(rng, -120, 120, false); + case OP_SIGMOID: + case OP_SILU: + case OP_GELU: return rand_range(rng, -20.0f, 20.0f); + default: return 0.0f; + } +} + +// Raw-op accuracy against libm, with the bounds the MUFU spec guarantees. +static bool raw_accurate(int op, float x, float y) { + double r; + switch (op) { + case OP_RCP: r = 1.0 / x; break; + case OP_RSQRT: r = 1.0 / std::sqrt((double)x); break; + case OP_EX2: r = std::exp2((double)x); break; + case OP_LG2: r = std::log2((double)x); break; + case OP_SINPI: r = std::sin(M_PI * std::fmod((double)x, 2.0)); break; + case OP_COSPI: r = std::cos(M_PI * std::fmod((double)x, 2.0)); break; + case OP_TANH: r = std::tanh((double)x); break; + default: return true; + } + // Out of scope for the bound: specials, and results the FTZ/overflow rules decide. + if (!std::isnormal(x) || std::isnan(r) || std::isinf(r) + || std::fabs(r) >= 3.4028234663852886e38 || std::fabs(r) < 1.1754943508222875e-38) + return true; + double err = std::fabs((double)y - r); + int ex; + std::frexp(r, &ex); + double ulp = std::ldexp(1.0, ex - 24); + switch (op) { + case OP_LG2: return (x >= 0.5f && x <= 2.0f) ? err <= 0x1p-22 : err <= 2 * ulp; + case OP_SINPI: + case OP_COSPI: return err <= 0x1p-21; + case OP_TANH: return err <= 0x1p-11 * std::fabs(r); + default: return err <= 2 * ulp; + } +} + +// Composite reference and tolerance; the composition rounds its prescale, so +// the bound grows with |x| where the prescale is a multiply. +static bool composite_accurate(int op, float x, float y) { + double xd = x, r, tol; + switch (op) { + case OP_EXPF: r = std::exp(xd); tol = (2e-7 + 1e-7 * std::fabs(xd)) * std::fabs(r); break; + case OP_LOGF: r = std::log(xd); tol = 1e-6 + 4e-7 * std::fabs(r); break; + case OP_SINF: r = std::sin(xd); tol = 1e-6 + 1e-7 * std::fabs(xd); break; + case OP_COSF: r = std::cos(xd); tol = 1e-6 + 1e-7 * std::fabs(xd); break; + case OP_SQRTF: r = std::sqrt(xd); tol = 4e-7 * r; break; + // tanh is accurate to 2^-11 relative; 1/2 + tanh/2 turns that into 2^-12 + // absolute (small sigmoid values are not relatively accurate). + case OP_SIGMOID: r = 1.0 / (1.0 + std::exp(-xd)); tol = 0x1p-12; break; + case OP_SILU: r = xd / (1.0 + std::exp(-xd)); tol = 0x1p-12 * std::max(1.0, std::fabs(xd)); break; + case OP_GELU: r = 0.5 * xd * (1.0 + std::tanh(0.7978845608028654 * (xd + 0.044715 * xd * xd * xd))); + tol = 0x1p-12 * std::max(1.0, std::fabs(xd)); break; + default: return true; + } + return std::fabs((double)y - r) <= tol; +} + +static int run_op(int op, std::mt19937& rng, uint64_t src_addr, uint64_t dst_addr) { + std::vector h_src(num_points); + std::vector h_dst(num_points); + size_t ns = (op <= OP_TANH) ? std::min(num_points, sizeof(kSpecials) / sizeof(kSpecials[0])) : 0; + for (uint32_t i = 0; i < num_points; ++i) + h_src[i] = (i < ns) ? as_float(kSpecials[i]) : gen_input(op, rng); + + kernel_arg_t arg = {}; + arg.op = op; + arg.num_points = num_points; + arg.repeat = repeat; + arg.src_addr = src_addr; + arg.dst_addr = dst_addr; + + uint32_t buf_size = num_points * sizeof(float); + RT_CHECK(vx_enqueue_write(queue, src_buffer, 0, h_src.data(), buf_size, 0, nullptr, nullptr)); + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &num_points, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + RT_CHECK(vx_enqueue_read(queue, h_dst.data(), dst_buffer, 0, buf_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + for (uint32_t i = 0; i < num_points; ++i) { + float x = h_src[i], y = h_dst[i]; + bool ok; + if (op <= OP_TANH) { + uint32_t ref = vortex::mufu::eval(op, as_bits(x)); + for (uint32_t r = 1; r < repeat; ++r) + ref = vortex::mufu::eval(op, ref); + ok = (as_bits(y) == ref); + if (ok && repeat == 1) + ok = raw_accurate(op, x, y); + if (!ok && errors < 20) + printf("*** error: %s[%u] x=%a (0x%08x): actual=0x%08x expected=0x%08x\n", + kOpNames[op], i, x, as_bits(x), as_bits(y), ref); + } else { + ok = (repeat > 1) || composite_accurate(op, x, y); + if (!ok && errors < 20) + printf("*** error: %s[%u] x=%a: actual=%a\n", kOpNames[op], i, x, y); + } + errors += !ok; + } + std::cout << kOpNames[op] << ": " << num_points << " points, " + << errors << " errors" << std::endl; + return errors; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t isa_flags; + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + if ((isa_flags & VX_ISA_EXT_MUFU) == 0) { + std::cout << "MUFU extension not supported!" << std::endl; + cleanup(); + return -1; + } + + uint32_t buf_size = num_points * sizeof(float); + uint64_t src_addr, dst_addr; + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_READ, &src_buffer)); + RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_WRITE, &dst_buffer)); + RT_CHECK(vx_buffer_address(dst_buffer, &dst_addr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::mt19937 rng(0x6d756675); + int errors = 0; + for (int op = 0; op < OP_COUNT; ++op) { + if (selected_op >= 0 && op != selected_op) + continue; + errors += run_op(op, rng, src_addr, dst_addr); + } + + std::cout << "cleanup" << std::endl; + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} From 7dddd7647dffb7c2b3db87df1ffa01594c7626b3 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 05:31:38 -0700 Subject: [PATCH 2/9] csr: write the first active lane's operand --- hw/rtl/core/VX_csr_unit.sv | 13 +++++++++++-- sim/simx/csr_unit.cpp | 40 ++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 925d1e7cff..e6e2f70334 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -68,12 +68,21 @@ module VX_csr_unit import VX_gpu_pkg::*; #( assign execute_if.ready = csr_req_ready && cta_read_done; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs1_data; - `UNUSED_VAR (rs1_data) for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rs1_data assign rs1_data[i] = execute_if.data.rs1_data[i]; end - wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_data[0]; + reg [`VX_CFG_XLEN-1:0] rs1_first; + always @(*) begin + rs1_first = rs1_data[0]; + for (integer i = NUM_LANES-1; i >= 0; --i) begin + if (execute_if.data.header.tmask[i]) begin + rs1_first = rs1_data[i]; + end + end + end + + wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_first; wire [`VX_CFG_XLEN-1:0] csr_scalar_data, dcr_read_data; diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 8bda8d2a36..74d9b8ea72 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -414,36 +414,30 @@ void CsrUnit::process(instr_trace_t* trace) { trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; + if (thread_start == num_threads) + return; + + // CSRs are per warp: every active lane reads the old value, and the write + // takes the first active lane's operand, as in the RTL. + for (uint32_t t = thread_start; t < num_threads; ++t) { + if (!tmask.test(t)) continue; + rd_data[t].i = this->get_csr(csr_addr, wid, t); + } + Word csr_value = rd_data[thread_start].i; + Word src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[thread_start].i; + switch (csr_type) { case CsrType::CSRRW: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; - this->set_csr(csr_addr, src_v, wid, t); - rd_data[t].i = csr_value; - } + this->set_csr(csr_addr, src_v, wid, thread_start); break; case CsrType::CSRRS: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; - if (src_v != 0) { - this->set_csr(csr_addr, csr_value | src_v, wid, t); - } - rd_data[t].i = csr_value; + if (src_v != 0) { + this->set_csr(csr_addr, csr_value | src_v, wid, thread_start); } break; case CsrType::CSRRC: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; - if (src_v != 0) { - this->set_csr(csr_addr, csr_value & ~src_v, wid, t); - } - rd_data[t].i = csr_value; + if (src_v != 0) { + this->set_csr(csr_addr, csr_value & ~src_v, wid, thread_start); } break; default: From 0ed20dc30601e8710712f500c07f4f57b3e100b0 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 05:31:38 -0700 Subject: [PATCH 3/9] zfhmin: half-precision load/store, move and conversion --- VX_config.toml | 5 +- ci/testcases/zfhmin.yaml | 58 +++++ docs/designs/floating_point_unit.md | 31 ++- hw/dpi/dpi_float.cpp | 10 +- hw/rtl/VX_gpu_pkg.sv | 1 + hw/rtl/VX_trace_pkg.sv | 16 +- hw/rtl/core/VX_decode.sv | 20 +- hw/rtl/core/VX_lsu_slice.sv | 9 +- hw/rtl/fpu/VX_fcvt_unit.sv | 241 +++++++++++++------ hw/rtl/fpu/VX_fncp_unit.sv | 24 +- hw/rtl/fpu/VX_fpu_dpi.sv | 19 +- hw/rtl/fpu/VX_fpu_dsp.sv | 12 +- hw/rtl/fpu/VX_fpu_fpnew.sv | 44 +++- hw/rtl/fpu/VX_fpu_std.sv | 12 +- hw/unittest/fcvt_unit/Makefile | 6 + hw/unittest/fcvt_unit/main.cpp | 44 +++- sim/simx/decode.cpp | 49 +++- sim/simx/fpu_unit.cpp | 36 +++ sim/simx/lsu_unit.cpp | 11 +- sim/simx/types.h | 1 + sw/common/rvfloats.cpp | 14 ++ sw/common/rvfloats.h | 2 + sw/runtime/include/vortex2.h | 1 + tests/regression/Makefile | 2 +- tests/regression/common.mk | 10 +- tests/regression/zfhmin/Makefile | 20 ++ tests/regression/zfhmin/common.h | 48 ++++ tests/regression/zfhmin/kernel.cpp | 180 ++++++++++++++ tests/regression/zfhmin/main.cpp | 360 ++++++++++++++++++++++++++++ 29 files changed, 1146 insertions(+), 140 deletions(-) create mode 100644 ci/testcases/zfhmin.yaml create mode 100644 tests/regression/zfhmin/Makefile create mode 100644 tests/regression/zfhmin/common.h create mode 100644 tests/regression/zfhmin/kernel.cpp create mode 100644 tests/regression/zfhmin/main.cpp diff --git a/VX_config.toml b/VX_config.toml index f999fad140..2e3004e6f4 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -34,6 +34,9 @@ VX_CFG_EXT_A_ENABLE = false VX_CFG_EXT_ZACAS_ENABLE = false VX_CFG_EXT_ZICOND_ENABLE = true +# Zfhmin: half-precision load/store, move and fp16<->fp32/fp64 conversion (no +# half arithmetic). Requires F. +VX_CFG_EXT_ZFHMIN_ENABLE = false VX_CFG_EXT_TCU_ENABLE = false VX_CFG_EXT_DMA_ENABLE = false VX_CFG_EXT_DXA_ENABLE = false @@ -455,7 +458,7 @@ VX_CFG_NUM_OCACHES = "expr: max(1, up($VX_CFG_NUM_OM_CORES / 4))" VX_CFG_MISA_STD = "expr: ($VX_CFG_EXT_A_ENABLED << 0) | (0 << 1) | ($VX_CFG_EXT_C_ENABLED << 2) | ($VX_CFG_EXT_D_ENABLED << 3) | (0 << 4) | ($VX_CFG_EXT_F_ENABLED << 5) | (0 << 6) | (0 << 7) | (1 << 8) | (0 << 9) | (0 << 10) | (0 << 11) | ($VX_CFG_EXT_M_ENABLED << 12) | (0 << 13) | (0 << 14) | (0 << 15) | (0 << 16) | (0 << 17) | (0 << 18) | (0 << 19) | (1 << 20) | (0 << 21) | (0 << 22) | (1 << 23) | (0 << 24) | (0 << 25)" # Custom extensions signature -VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11) | ($VX_CFG_EXT_MUFU_ENABLED << 12)" +VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11) | ($VX_CFG_EXT_MUFU_ENABLED << 12) | ($VX_CFG_EXT_ZFHMIN_ENABLED << 13)" [toolchain] ASIC = false diff --git a/ci/testcases/zfhmin.yaml b/ci/testcases/zfhmin.yaml new file mode 100644 index 0000000000..7e150cb385 --- /dev/null +++ b/ci/testcases/zfhmin.yaml @@ -0,0 +1,58 @@ +# Zfhmin: half-precision load/store, move and fp16<->fp32/fp64 conversion. +# The `zfhmin` app checks every conversion bit-exactly against softfloat, with +# fflags, across all rounding modes, exhaustively for the widening direction, +# plus flh/fsh/fmv NaN-boxing. The converter datapath is also covered by +# unittest:hw-fpu (fcvt_unit). +category: zfhmin +defaults: + xlen: + - 32 + - 64 + tier: smoke + touches: [hw/rtl/fpu, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_slice.sv, hw/rtl/core/VX_csr_unit.sv, hw/dpi/dpi_float.cpp, sim/simx/fpu_unit.cpp, sim/simx/decode.cpp, sim/simx/lsu_unit.cpp, sim/simx/csr_unit.cpp, sw/common/rvfloats.cpp] +tests: +- id: zfhmin-1 + via: blackbox + drivers: + - simx + - rtlsim + app: zfhmin + args: -n1024 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE +# The synthesizable converter (STD) rather than the DPI model rtlsim uses by default. +- id: zfhmin-std + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n256 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_FPU_TYPE_STD +- id: zfhmin-dsp + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n256 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_FPU_TYPE_DSP + xlen: + - 32 +# Wide warp with a single FPU lane: conversions split into one packet per thread. +- id: zfhmin-nt16-fpu1 + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n0 -p0x23 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_FPU_LANES=1 + xlen: + - 32 + +# --------------------------------------------------------------------------- +# check: model_parity — SimX vs rtlsim on the narrowing/widening/boxing phases. +# --------------------------------------------------------------------------- +- id: parity-zfhmin + check: model_parity + via: blackbox + app: zfhmin + args: -n1024 -p0x23 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index 56fd16c78d..a4e2fda7fd 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -85,11 +85,14 @@ Decode emits a 4-bit operation, a 2-bit `fmt`, and a 3-bit `frm` | `1000` / `1001` | `F2I` / `F2U` | CVT | `[0]` float is double, `[1]` integer is 64-bit | rounding mode | | `1010` / `1011` | `I2F` / `U2F` | CVT | `[0]` float is double, `[1]` integer is 64-bit | rounding mode | | `1100` | `CMP` | NCP | `[0]` double | `LE` = 0, `LT` = 1, `EQ` = 2 | -| `1101` | `F2F` | **CVT** | `[0]` destination is double | rounding mode | -| `1110` | `MISC` | NCP | `[0]` double | `SGNJ`, `SGNJN`, `SGNJX`, `CLASS`, `MVXW`, `MVWX`, `FMIN`, `FMAX` = 0–7 | +| `1101` | `F2F` | **CVT** | `[0]` destination is double, `[1]` source is half (Zfhmin) | rounding mode | +| `1110` | `MISC` | NCP | `[0]` double, `[1]` half for `MVXW`/`MVWX` (Zfhmin) | `SGNJ`, `SGNJN`, `SGNJX`, `CLASS`, `MVXW`, `MVWX`, `FMIN`, `FMAX` = 0–7 | +| `1111` | `F2H` | **CVT** | `[0]` source is double | rounding mode | -`F2F` is the one exception to the index rule: its top bits say NCP, and the -request switch steers it to CVT. +`F2F` and `F2H` are the exceptions to the index rule: their top bits say NCP, +and the request switch steers them to CVT. `F2H` (`FCVT.H.S`, `FCVT.H.D`) and +the half forms of `F2F` and the moves exist only with +`VX_CFG_EXT_ZFHMIN_ENABLE`. Subtraction is not an operation. `SUB`, `MSUB` and `NMSUB` are `ADD`, `MADD` and `NMADD` with `fmt[1]` set, and the FMA core folds the sign into its @@ -291,9 +294,15 @@ value sized to the widest enabled format. - **Unpack** selects the source format's fields, re-biases, and left-aligns. - **Round and pack** narrows the canonical value to the destination through one shared tree, reused for every format pair. -- **Narrowing** (`FCVT.S.D`) handles a subnormal destination by - denormalizing and rounding with a correct guard/round/sticky split, and - overflow to infinity. +- **Narrowing** (`FCVT.S.D`, and `FCVT.H.S`/`FCVT.H.D` with Zfhmin) handles a + subnormal destination by denormalizing and rounding with a correct + guard/round/sticky split; a subnormal that rounds up becomes the smallest + normal. Underflow uses RISC-V's after-rounding tininess. Overflow yields + infinity or the largest finite value according to the rounding mode, and a + rounding carry into the infinity exponent also raises OF. +- **Half** (`HALF_ENABLE`, set by `VX_CFG_EXT_ZFHMIN_ENABLE`) adds an F16 + source and destination; a half source that is not NaN-boxed reads as the + canonical NaN. - The integer side is `XLEN` wide: I32, I64, U32, U64. Latency is `VX_CFG_FCVT_LATENCY` on a 32-bit build and **one more on a @@ -431,7 +440,8 @@ flight, and a result waiting on the commit arbiter keeps its tag: | what | where | checks | |---|---|---| -| leaf units | `hw/unittest/{fma,fdiv,fsqrt,fcvt,fdivsqrt}_unit`, run by `make -C hw/unittest run-fpu` | bit-exact result and flags against soft-float, over every RISC-V rounding mode, in the default and the area-reduced configurations | +| leaf units | `hw/unittest/{fma,fdiv,fsqrt,fcvt,fdivsqrt}_unit`, run by `make -C hw/unittest run-fpu` | bit-exact result and flags against soft-float, over every RISC-V rounding mode, in the default and the area-reduced configurations (`fcvt` includes F32↔F16) | +| Zfhmin | `zfhmin` category: the `zfhmin` app on simx and rtlsim, with the DPI, STD and DSP backends | every half conversion bit-exact with flags across all rounding modes (widening exhaustive), `flh`/`fsh`/`fmv` NaN-boxing | | FPU RTL end to end | `riscv` category, `isa-32f-dsp`: `rtlsim` rebuilt with the DSP backend, running `rv32uf` | the soft cores under the full pipeline, F32 | | ISA conformance | `riscv` category, every other lane: `rv32uf`, `rv64uf`, `rv64ud` | decode, the front end and writeback — arithmetic comes from the DPI model | | STD backend on hardware | `config2` category, `dogfood` with `-DVX_CFG_FPU_TYPE_STD` on `xrt` | the soft cores on the FPGA | @@ -468,3 +478,8 @@ different warps execute concurrently; it ignores `frm` and never writes models `VX_CFG_FCVT_LATENCY` on every build, one cycle short. - **Double precision on a 32-bit build** (§4.1) is not supported by the operand path. +- **FPNEW narrowing of infinity.** The pinned cvfpu treats an infinite F2F + source as an overflow, so `RTZ`, and `RDN`/`RUP` toward zero, return the + largest finite value instead of infinity (`FCVT.S.D`, `FCVT.H.S`, + `FCVT.H.D`), and `FCVT.H.S` of a signaling NaN does not raise NV. The DPI, + STD and DSP backends are correct. diff --git a/hw/dpi/dpi_float.cpp b/hw/dpi/dpi_float.cpp index b29dfe51fc..20ae399c81 100644 --- a/hw/dpi/dpi_float.cpp +++ b/hw/dpi/dpi_float.cpp @@ -63,7 +63,7 @@ inline uint64_t nan_box16(uint32_t value) { #ifdef VX_CFG_XLEN_64 return value | 0xffffffffffff0000; #else - return value; + return (value & 0xffff) | 0xffff0000; #endif } @@ -103,7 +103,7 @@ inline bool is_nan_boxed16(uint64_t value) { #ifdef VX_CFG_XLEN_64 return (uint64_t(value >> 16) == 0xffffffffffff); #else - return true; + return ((uint32_t(value) >> 16) == 0xffff); #endif } @@ -360,6 +360,9 @@ void dpi_f2f(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal case 0: // fp32 *result = rv_ftod(check_boxing(a), *frm, fflags); break; + case 2: // fp16 + *result = rv_htod(check_boxing16(a, 0x7E00), *frm, fflags); + break; default: std::abort(); } @@ -369,6 +372,9 @@ void dpi_f2f(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal case 0: // fp32 *result = nan_box16(rv_ftoh_s(check_boxing(a), *frm, fflags)); break; + case 1: // fp64 + *result = nan_box16(rv_dtoh(a, *frm, fflags)); + break; default: std::abort(); } diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 73a49bf371..b66e7ac049 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -490,6 +490,7 @@ package VX_gpu_pkg; localparam INST_FPU_CMP = 4'b1100; // frm: LE=0, LT=1, EQ=2 localparam INST_FPU_F2F = 4'b1101; // fmt[0]: F32=0, F64=1 localparam INST_FPU_MISC = 4'b1110; // frm: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7 + localparam INST_FPU_F2H = 4'b1111; localparam INST_FPU_BITS = 4; function automatic logic inst_fpu_is_class(input logic [INST_FPU_BITS-1:0] op, input logic [INST_FRM_BITS-1:0] frm); diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index 39fb310cec..cc2d753f56 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -363,10 +363,18 @@ package VX_trace_pkg; end end INST_FPU_F2F: begin + case (op_args.fpu.fmt) + 2'b00: `TRACE(level, ("FCVT.S.D")) + 2'b01: `TRACE(level, ("FCVT.D.S")) + 2'b10: `TRACE(level, ("FCVT.S.H")) + 2'b11: `TRACE(level, ("FCVT.D.H")) + endcase + end + INST_FPU_F2H: begin if (op_args.fpu.fmt[0]) begin - `TRACE(level, ("FCVT.D.S")) + `TRACE(level, ("FCVT.H.D")) end else begin - `TRACE(level, ("FCVT.S.D")) + `TRACE(level, ("FCVT.H.S")) end end INST_FPU_F2I: begin @@ -419,8 +427,8 @@ package VX_trace_pkg; 1: `TRACE(level, ("FSGNJN.S")) 2: `TRACE(level, ("FSGNJX.S")) 3: `TRACE(level, ("FCLASS.S")) - 4: `TRACE(level, ("FMV.X.S")) - 5: `TRACE(level, ("FMV.S.X")) + 4: `TRACE(level, (op_args.fpu.fmt[1] ? "FMV.X.H" : "FMV.X.S")) + 5: `TRACE(level, (op_args.fpu.fmt[1] ? "FMV.H.X" : "FMV.S.X")) 6: `TRACE(level, ("FMIN.S")) 7: `TRACE(level, ("FMAX.S")) endcase diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 25cbd3dcdd..f5775817fe 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -506,7 +506,19 @@ module VX_decode import `USED_FREG (rs1); `USED_FREG (rs2); end - `ifdef VX_CFG_FLEN_64 + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + 5'b01000: begin + if (funct2 == 2'b10) begin + op_type = INST_OP_BITS'(INST_FPU_F2H); + op_args.fpu.fmt[0] = rs2[0]; + end else begin + op_type = INST_OP_BITS'(INST_FPU_F2F); + end + rd_xregs[XREG_1] = frm_is_dyn; + `USED_FREG (rd); + `USED_FREG (rs1); + end + `elsif VX_CFG_FLEN_64 5'b01000: begin // FCVT.S.D, FCVT.D.S op_type = INST_OP_BITS'(INST_FPU_F2F); @@ -560,6 +572,9 @@ module VX_decode import // NCP: FMV.X.W=4 op_type = INST_OP_BITS'(INST_FPU_MISC); op_args.fpu.frm = INST_FRM_BITS'(4); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + op_args.fpu.fmt[1] = funct2[1]; + `endif end wr_xregs[XREG_0] = 1'b0; `USED_IREG (rd); @@ -569,6 +584,9 @@ module VX_decode import // NCP: FMV.W.X=5 op_type = INST_OP_BITS'(INST_FPU_MISC); op_args.fpu.frm = INST_FRM_BITS'(5); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + op_args.fpu.fmt[1] = funct2[1]; + `endif wr_xregs[XREG_0] = 1'b0; `USED_FREG (rd); `USED_IREG (rs1); diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index affb932298..5a54698dd6 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -413,13 +413,16 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rsp_data; -`ifdef VX_CFG_XLEN_64 `ifdef VX_CFG_EXT_F_ENABLE // apply nan-boxing to flw outputs wire rsp_is_float = (get_reg_type(rsp_hdr.rd) == REG_TYPE_F); `else wire rsp_is_float = 0; `endif +`ifndef VX_CFG_XLEN_64 +`ifndef VX_CFG_EXT_ZFHMIN_ENABLE + `UNUSED_VAR (rsp_is_float) +`endif `endif for (genvar i = 0; i < NUM_LANES; i++) begin : g_rsp_data @@ -435,7 +438,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( always @(*) begin case (inst_lsu_fmt(rsp_op_type)) LSU_FMT_B: rsp_data[i] = `VX_CFG_XLEN'(signed'(rsp_data8)); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + LSU_FMT_H: rsp_data[i] = rsp_is_float ? {{(`VX_CFG_XLEN-16){1'b1}}, rsp_data16} : `VX_CFG_XLEN'(signed'(rsp_data16)); + `else LSU_FMT_H: rsp_data[i] = `VX_CFG_XLEN'(signed'(rsp_data16)); + `endif LSU_FMT_BU: rsp_data[i] = `VX_CFG_XLEN'(unsigned'(rsp_data8)); LSU_FMT_HU: rsp_data[i] = `VX_CFG_XLEN'(unsigned'(rsp_data16)); `ifdef VX_CFG_XLEN_64 diff --git a/hw/rtl/fpu/VX_fcvt_unit.sv b/hw/rtl/fpu/VX_fcvt_unit.sv index 8cbeeed93e..22ef55e3e6 100644 --- a/hw/rtl/fpu/VX_fcvt_unit.sv +++ b/hw/rtl/fpu/VX_fcvt_unit.sv @@ -28,7 +28,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // sources to signed zero) for area. Use 0 for relaxed paths. parameter SUBNORM_ENABLE = 1, // 1: produce fflags (NV/OF/UF/NX). 0: tie fflags to 0 (area). - parameter EXCEPT_ENABLE = 1 + parameter EXCEPT_ENABLE = 1, + parameter HALF_ENABLE = 0 ) ( input wire clk, input wire reset, @@ -45,6 +46,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; input wire is_int64, // integer operand width: 1=I64, 0=I32 input wire src_fmt, // source float format: 1=F64, 0=F32 input wire dst_fmt, // dest float format: 1=F64, 0=F32 + input wire src_h, + input wire dst_h, input wire [`VX_CFG_XLEN-1:0] dataa, output wire [`VX_CFG_XLEN-1:0] result, @@ -53,7 +56,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; ); localparam F32_EXP = 8, F32_MAN = 23, F32_BIAS = 127; localparam F64_EXP = 11, F64_MAN = 52, F64_BIAS = 1023; + localparam F16_EXP = 5, F16_MAN = 10, F16_BIAS = 15; localparam HAS_D = (FLEN >= 64); + localparam HAS_H = (HALF_ENABLE != 0); // Widest enabled float format drives the canonical internal datapath. localparam SUPER_MAN = HAS_D ? F64_MAN : F32_MAN; @@ -67,9 +72,13 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; `UNUSED_VAR (src_fmt) `UNUSED_VAR (dst_fmt) `UNUSED_VAR (is_f2f) - wire src_is_d = HAS_D & src_fmt; - wire dst_is_d = HAS_D & dst_fmt; - wire f2f = HAS_D & is_f2f; + `UNUSED_VAR (src_h) + `UNUSED_VAR (dst_h) + wire src_is_h = HAS_H & src_h; + wire dst_is_h = HAS_H & dst_h; + wire src_is_d = HAS_D & src_fmt & ~src_is_h; + wire dst_is_d = HAS_D & dst_fmt & ~dst_is_h; + wire f2f = (HAS_D | HAS_H) & is_f2f; reg [LATENCY-1:0] mask_pipe; always @(posedge clk) begin @@ -88,8 +97,10 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [63:0] safe_dataa = 64'(dataa); + wire [15:0] h_op = (&dataa[`VX_CFG_XLEN-1:16]) ? dataa[15:0] : 16'h7E00; + // Float source field extraction (per source format) - wire input_fp_sgn = src_is_d ? safe_dataa[63] : safe_dataa[31]; + wire input_fp_sgn = src_is_h ? h_op[15] : src_is_d ? safe_dataa[63] : safe_dataa[31]; fclass_t fclass32; VX_fp_classifier #(.EXP_BITS(F32_EXP), .MAN_BITS(F32_MAN)) fp_classifier32 ( @@ -98,7 +109,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; .clss_o (fclass32) ); - fclass_t fclass; + fclass_t fclass_sd; if (HAS_D) begin : g_classify_d fclass_t fclass64; VX_fp_classifier #(.EXP_BITS(F64_EXP), .MAN_BITS(F64_MAN)) fp_classifier64 ( @@ -106,9 +117,23 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; .man_i (safe_dataa[F64_MAN-1:0]), .clss_o (fclass64) ); - assign fclass = src_is_d ? fclass64 : fclass32; + assign fclass_sd = src_is_d ? fclass64 : fclass32; end else begin : g_classify_s - assign fclass = fclass32; + assign fclass_sd = fclass32; + end + + fclass_t fclass; + if (HAS_H) begin : g_classify_h + fclass_t fclass16; + VX_fp_classifier #(.EXP_BITS(F16_EXP), .MAN_BITS(F16_MAN)) fp_classifier16 ( + .exp_i (h_op[F16_MAN +: F16_EXP]), + .man_i (h_op[F16_MAN-1:0]), + .clss_o (fclass16) + ); + assign fclass = src_is_h ? fclass16 : fclass_sd; + end else begin : g_classify_no_h + `UNUSED_VAR (h_op) + assign fclass = fclass_sd; end // Source significand {hidden, mantissa} placed in the low bits, and the @@ -117,17 +142,20 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // signed zero (mantissa forced 0 -> downstream zero path). wire src_daz = (SUBNORM_ENABLE == 0) & fclass.is_subnormal; wire [S_MAN_WIDTH-1:0] fp_unpacked_mant = src_daz ? '0 + : src_is_h ? S_MAN_WIDTH'({fclass.is_normal, h_op[F16_MAN-1:0]}) : src_is_d ? S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F64_MAN-1:0]}) : S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F32_MAN-1:0]}); - wire signed [S_EXP_WIDTH-1:0] src_bias_s = src_is_d ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); - wire [SUPER_EXP-1:0] src_exp_raw; + wire signed [S_EXP_WIDTH-1:0] src_bias_s = src_is_h ? S_EXP_WIDTH'(F16_BIAS) + : src_is_d ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); + wire [SUPER_EXP-1:0] src_exp_sd; if (HAS_D) begin : g_srcexp_d - assign src_exp_raw = src_is_d ? safe_dataa[F64_MAN +: F64_EXP] - : SUPER_EXP'(safe_dataa[F32_MAN +: F32_EXP]); + assign src_exp_sd = src_is_d ? safe_dataa[F64_MAN +: F64_EXP] + : SUPER_EXP'(safe_dataa[F32_MAN +: F32_EXP]); end else begin : g_srcexp_s - assign src_exp_raw = safe_dataa[F32_MAN +: F32_EXP]; + assign src_exp_sd = safe_dataa[F32_MAN +: F32_EXP]; end + wire [SUPER_EXP-1:0] src_exp_raw = src_is_h ? SUPER_EXP'(h_op[F16_MAN +: F16_EXP]) : src_exp_sd; wire i_sign = safe_dataa[is_int64 ? (`VX_CFG_XLEN-1) : 31] && is_signed; wire [`VX_CFG_XLEN-1:0] i_mag_raw = i_sign ? (-dataa) : dataa; @@ -149,21 +177,21 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; end // Stage 0 -> Stage 1 - wire is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, src_d_s1, dst_d_s1; + wire is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1; wire [INST_FRM_BITS-1:0] frm_s1; fclass_t fclass_s1; wire [S_MAN_WIDTH-1:0] unpacked_mant_s1; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s1; VX_pipe_register #( - .DATAW (8 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH), + .DATAW (10 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH), .DEPTH (LATENCY > 5) ) pipe_reg1 ( .clk (clk), .reset (reset), .enable (enable && mask), - .data_in ({is_itof, is_ftoi, f2f, is_signed, is_int64, src_is_d, dst_is_d, input_sign_s0, frm, fclass, unpacked_mant_s0, unpacked_exp_s0}), - .data_out ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, src_d_s1, dst_d_s1, input_sign_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1}) + .data_in ({is_itof, is_ftoi, f2f, is_signed, is_int64, src_is_d, dst_is_d, src_is_h, dst_is_h, input_sign_s0, frm, fclass, unpacked_mant_s0, unpacked_exp_s0}), + .data_out ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1, input_sign_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1}) ); // ====================================================================== @@ -226,7 +254,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire mant_is_zero_s1 = ~mant_is_nonzero_s1; // Stage 1 -> Stage 2 - wire is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2; + wire is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, src_h_s2, dst_h_s2; wire [INST_FRM_BITS-1:0] frm_s2; fclass_t fclass_s2; wire [S_MAN_WIDTH-1:0] unpacked_mant_s2; @@ -234,14 +262,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [LZC_RESULT_WIDTH-1:0] renorm_shamt_s2; VX_pipe_register #( - .DATAW (9 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + LZC_RESULT_WIDTH), + .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + LZC_RESULT_WIDTH), .DEPTH (LATENCY > 4) ) pipe_reg2 ( .clk (clk), .reset (reset), .enable (enable && stg2_mask), - .data_in ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, mant_is_zero_s1, src_d_s1, dst_d_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1, renorm_shamt_s1}), - .data_out ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, frm_s2, fclass_s2, unpacked_mant_s2, unpacked_exp_s2, renorm_shamt_s2}) + .data_in ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, mant_is_zero_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1, renorm_shamt_s1}), + .data_out ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, src_h_s2, dst_h_s2, frm_s2, fclass_s2, unpacked_mant_s2, unpacked_exp_s2, renorm_shamt_s2}) ); // ====================================================================== @@ -252,14 +280,15 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // the S_MAN_WIDTH field. LZC counts the high padding too: subtract it back // for float inputs, using the source format's mantissa width. wire [S_EXP_WIDTH-1:0] fp_mant_offset_s2 = S_EXP_WIDTH'(S_MAN_WIDTH - 1) - - (src_d_s2 ? S_EXP_WIDTH'(F64_MAN) : S_EXP_WIDTH'(F32_MAN)); + - (src_h_s2 ? S_EXP_WIDTH'(F16_MAN) : src_d_s2 ? S_EXP_WIDTH'(F64_MAN) : S_EXP_WIDTH'(F32_MAN)); + `UNUSED_VAR (src_d_s2) wire [S_MAN_WIDTH-1:0] norm_mant_s2 = unpacked_mant_s2 << renorm_shamt_s2; wire signed [S_EXP_WIDTH-1:0] norm_exp_s2 = unpacked_exp_s2 - S_EXP_WIDTH'({1'b0, renorm_shamt_s2}) + (is_itof_s2 ? S_EXP_WIDTH'(0) : fp_mant_offset_s2); // Stage 2 -> Stage 3 - wire is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3; + wire is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3; wire [INST_FRM_BITS-1:0] frm_s3; fclass_t fclass_s3; wire [S_MAN_WIDTH-1:0] norm_mant_s3; @@ -267,14 +296,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s3; // pre-renorm exp, needed for f2i_shamt VX_pipe_register #( - .DATAW (9 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + 2*S_EXP_WIDTH), + .DATAW (10 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + 2*S_EXP_WIDTH), .DEPTH (LATENCY > 3) ) pipe_reg3 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-5]), - .data_in ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, frm_s2, fclass_s2, norm_mant_s2, norm_exp_s2, unpacked_exp_s2}), - .data_out ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, frm_s3, fclass_s3, norm_mant_s3, norm_exp_s3, unpacked_exp_s3}) + .data_in ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, dst_h_s2, frm_s2, fclass_s2, norm_mant_s2, norm_exp_s2, unpacked_exp_s2}), + .data_out ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3, frm_s3, fclass_s3, norm_mant_s3, norm_exp_s3, unpacked_exp_s3}) ); // ====================================================================== @@ -285,15 +314,18 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] f2i_shamt_s3 = S_EXP_WIDTH'(S_MAN_WIDTH - 1) - unpacked_exp_s3; // Destination float bias and true biased destination exponent. - wire signed [S_EXP_WIDTH-1:0] dst_bias_s3 = dst_d_s3 ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); + wire signed [S_EXP_WIDTH-1:0] dst_bias_s3 = dst_h_s3 ? S_EXP_WIDTH'(F16_BIAS) + : dst_d_s3 ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); wire signed [S_EXP_WIDTH-1:0] dst_exp_s3 = norm_exp_s3 + dst_bias_s3; + wire signed [S_EXP_WIDTH-1:0] dst_inf_exp_s3 = dst_h_s3 ? S_EXP_WIDTH'(31) : S_EXP_WIDTH'(255); - // F2F narrowing (FCVT.S.D) can under/overflow the F32 range; widening - // (FCVT.D.S) is always exact, so it never does. Only the F32-dst finite - // nonzero case matters here. + // F2F narrowing (FCVT.S.D, FCVT.H.S, FCVT.H.D) can under/overflow the + // destination range; widening is always exact, so it never does. Only the + // F32/F16-dst finite nonzero case matters here. wire f2f_narrow_s3 = is_f2f_s3 & ~dst_d_s3 & ~fclass_s3.is_nan & ~fclass_s3.is_inf & ~fclass_s3.is_zero; wire f2f_uf_s3 = f2f_narrow_s3 & (dst_exp_s3 <= 0); // -> subnormal/zero - wire f2f_of_s3 = f2f_narrow_s3 & (dst_exp_s3 >= S_EXP_WIDTH'(255)); // -> overflow to inf + wire f2f_of_s3 = f2f_narrow_s3 & (dst_exp_s3 >= dst_inf_exp_s3); // -> overflow + wire f2f_tiny0_s3 = f2f_narrow_s3 & (dst_exp_s3 == 0); wire signed [S_EXP_WIDTH-1:0] denorm_sh_s3 = S_EXP_WIDTH'(1) - dst_exp_s3; // right-shift to denormalize // Alignment right-shift: F2I integer align, or F2F destination denormalization. @@ -312,8 +344,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [SUPER_EXP-1:0] final_exp_s3 = SUPER_EXP'(dst_exp_s3); // Stage 3 -> Stage 4 - wire is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4; - wire f2f_uf_s4, f2f_of_s4; + wire is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4; + wire f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4; wire [INST_FRM_BITS-1:0] frm_s4; fclass_t fclass_s4; wire [S_MAN_WIDTH-1:0] norm_mant_s4; @@ -322,14 +354,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s4; // true magnitude exponent (F2I overflow) VX_pipe_register #( - .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + SUPER_EXP + S_EXP_WIDTH), + .DATAW (13 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + SUPER_EXP + S_EXP_WIDTH), .DEPTH (LATENCY > 2) ) pipe_reg4 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-4]), - .data_in ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, f2f_uf_s3, f2f_of_s3, frm_s3, fclass_s3, norm_mant_s3, align_shamt_s3, final_exp_s3, unpacked_exp_s3}), - .data_out ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, f2f_uf_s4, f2f_of_s4, frm_s4, fclass_s4, norm_mant_s4, align_shamt_s4, final_exp_s4, unpacked_exp_s4}) + .data_in ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3, f2f_uf_s3, f2f_of_s3, f2f_tiny0_s3, frm_s3, fclass_s3, norm_mant_s3, align_shamt_s3, final_exp_s3, unpacked_exp_s3}), + .data_out ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4, f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4, frm_s4, fclass_s4, norm_mant_s4, align_shamt_s4, final_exp_s4, unpacked_exp_s4}) ); // ====================================================================== @@ -344,7 +376,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire sticky_bit_s4 = |aligned_mant_full_s4[S_MAN_WIDTH-2:0]; // Destination float mantissa width (I2F / F2F) - wire [5:0] dst_man_w_s4 = dst_d_s4 ? 6'(F64_MAN) : 6'(F32_MAN); + wire [5:0] dst_man_w_s4 = dst_h_s4 ? 6'(F16_MAN) : dst_d_s4 ? 6'(F64_MAN) : 6'(F32_MAN); // Kept significand = {hidden, dst_man}. The round (guard) bit is the first bit // BELOW the kept LSB; sticky ORs everything beneath the guard. (Folding the @@ -355,6 +387,12 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [S_MAN_WIDTH-1:0] fp_sticky_mask_s4= (S_MAN_WIDTH'(1) << (fp_trunc_sh_s4 - 7'd1)) - 1; wire sticky_red_s4 = |(aligned_mant_s4 & fp_sticky_mask_s4) | guard_bit_s4 | round_bit_s4 | sticky_bit_s4; + wire [S_MAN_WIDTH-1:0] fine_trunc_s4 = norm_mant_s4 >> fp_trunc_sh_s4; + wire [S_MAN_WIDTH-1:0] fine_ones_s4 = (S_MAN_WIDTH'(1) << (dst_man_w_s4 + 6'd1)) - 1; + wire [2:0] fine_round_s4 = {fine_trunc_s4 == fine_ones_s4, + norm_mant_s4[LZC_RESULT_WIDTH'(fp_trunc_sh_s4 - 7'd1)], + |(norm_mant_s4 & fp_sticky_mask_s4)}; + logic [1:0] round_sticky_bits_s4; logic [S_MAN_WIDTH-1:0] pre_round_abs_s4; @@ -371,8 +409,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; end // Stage 4 -> Stage 5 - wire is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5; - wire f2f_uf_s5, f2f_of_s5; + wire is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, dst_h_s5; + wire f2f_uf_s5, f2f_of_s5, f2f_tiny0_s5; + wire [2:0] fine_round_s5; wire [INST_FRM_BITS-1:0] frm_s5; fclass_t fclass_s5; wire [1:0] round_sticky_bits_s5; @@ -381,14 +420,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s5; VX_pipe_register #( - .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + 2 + S_MAN_WIDTH + SUPER_EXP + S_EXP_WIDTH), + .DATAW (16 + INST_FRM_BITS + $bits(fclass_t) + 2 + S_MAN_WIDTH + SUPER_EXP + S_EXP_WIDTH), .DEPTH (LATENCY > 1) ) pipe_reg5 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-3]), - .data_in ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, f2f_uf_s4, f2f_of_s4, frm_s4, fclass_s4, round_sticky_bits_s4, pre_round_abs_s4, final_exp_s4, unpacked_exp_s4}), - .data_out ({is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, f2f_uf_s5, f2f_of_s5, frm_s5, fclass_s5, round_sticky_bits_s5, pre_round_abs_s5, final_exp_s5, unpacked_exp_s5}) + .data_in ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4, f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4, fine_round_s4, frm_s4, fclass_s4, round_sticky_bits_s4, pre_round_abs_s4, final_exp_s4, unpacked_exp_s4}), + .data_out ({is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, dst_h_s5, f2f_uf_s5, f2f_of_s5, f2f_tiny0_s5, fine_round_s5, frm_s5, fclass_s5, round_sticky_bits_s5, pre_round_abs_s5, final_exp_s5, unpacked_exp_s5}) ); // ====================================================================== @@ -425,7 +464,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // ---- Float pack (I2F / F2F), per destination format ---- // F32 result, NaN-boxed when XLEN>32. F64 result fills the full width. - wire f32_man_ovf = rounded_abs_s5[F32_MAN + 1]; + wire f32_man_ovf = f2f_uf_s5 ? rounded_abs_s5[F32_MAN] : rounded_abs_s5[F32_MAN + 1]; wire [F32_EXP-1:0] fp32_exp = safe_dst_exp[F32_EXP-1:0] + F32_EXP'(f32_man_ovf); wire [F32_MAN-1:0] fp32_man = f32_man_ovf ? '0 : safe_rounded_abs[F32_MAN-1:0]; wire [31:0] fp_32_res = {rounded_sign_s5, fp32_exp, fp32_man}; @@ -434,52 +473,103 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; : `VX_CFG_XLEN'(fp_32_res); wire [`VX_CFG_XLEN-1:0] fp_dst_res; - wire f2f_nv_s5, f2f_nx_s5; - if (HAS_D) begin : g_fp_pack_d - wire f64_man_ovf = rounded_abs_s5[F64_MAN + 1]; - wire [F64_EXP-1:0] fp64_exp = safe_dst_exp[F64_EXP-1:0] + F64_EXP'(f64_man_ovf); - wire [F64_MAN-1:0] fp64_man = f64_man_ovf ? '0 : safe_rounded_abs[F64_MAN-1:0]; - wire [63:0] fp_64_res = {rounded_sign_s5, fp64_exp, fp64_man}; + wire f2f_nv_s5, f2f_nx_s5, f2f_of_out_s5, f2f_uf_out_s5; + if (HAS_D || HAS_H) begin : g_fp_pack_f2f + wire [`VX_CFG_XLEN-1:0] fp64_boxed; + if (HAS_D) begin : g_fp64 + wire f64_man_ovf = rounded_abs_s5[F64_MAN + 1]; + wire [F64_EXP-1:0] fp64_exp = safe_dst_exp[F64_EXP-1:0] + F64_EXP'(f64_man_ovf); + wire [F64_MAN-1:0] fp64_man = f64_man_ovf ? '0 : safe_rounded_abs[F64_MAN-1:0]; + assign fp64_boxed = `VX_CFG_XLEN'({rounded_sign_s5, fp64_exp, fp64_man}); + end else begin : g_no_fp64 + assign fp64_boxed = '0; + end + + wire f16_man_ovf = f2f_uf_s5 ? rounded_abs_s5[F16_MAN] : rounded_abs_s5[F16_MAN + 1]; + wire [F16_EXP-1:0] fp16_exp = safe_dst_exp[F16_EXP-1:0] + F16_EXP'(f16_man_ovf); + wire [F16_MAN-1:0] fp16_man = f16_man_ovf ? '0 : safe_rounded_abs[F16_MAN-1:0]; + wire [15:0] fp_16_res = {rounded_sign_s5, fp16_exp, fp16_man}; + + wire [`VX_CFG_XLEN-1:0] qnan_d = `VX_CFG_XLEN'(64'h7FF8000000000000); + wire [`VX_CFG_XLEN-1:0] inf_d = `VX_CFG_XLEN'({input_sign_s5, 11'h7FF, 52'd0}); + wire [`VX_CFG_XLEN-1:0] zero_d = `VX_CFG_XLEN'({input_sign_s5, 63'd0}); - // Canonical specials for F2F (NaN/Inf/Zero propagation in float domain) - wire [31:0] f2f_qnan_32 = 32'h7FC00000; - wire [31:0] f2f_inf_32 = {rounded_sign_s5, 8'hFF, 23'd0}; + wire [31:0] qnan_s32 = 32'h7FC00000; + wire [31:0] inf_s32 = {input_sign_s5, 8'hFF, 23'd0}; + wire [31:0] zero_s32 = {input_sign_s5, 31'd0}; + wire [31:0] max_s32 = {input_sign_s5, 8'hFE, {23{1'b1}}}; - wire [63:0] f2f_qnan_64 = 64'h7FF8000000000000; - wire [63:0] f2f_inf_64 = {rounded_sign_s5, 11'h7FF, 52'd0}; - wire [63:0] f2f_zero_64 = {input_sign_s5, 63'd0}; + wire [15:0] qnan_h16 = 16'h7E00; + wire [15:0] inf_h16 = {input_sign_s5, 5'h1F, 10'd0}; + wire [15:0] zero_h16 = {input_sign_s5, 15'd0}; + wire [15:0] max_h16 = {input_sign_s5, 5'h1E, {10{1'b1}}}; - // Normal (non-special) destination value by format. - wire [`VX_CFG_XLEN-1:0] normal_dst = dst_d_s5 ? `VX_CFG_XLEN'(fp_64_res) : f32_boxed; + wire [`VX_CFG_XLEN-1:0] box_s_qnan = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, qnan_s32} : `VX_CFG_XLEN'(qnan_s32); + wire [`VX_CFG_XLEN-1:0] box_s_inf = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, inf_s32} : `VX_CFG_XLEN'(inf_s32); + wire [`VX_CFG_XLEN-1:0] box_s_zero = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, zero_s32} : `VX_CFG_XLEN'(zero_s32); + wire [`VX_CFG_XLEN-1:0] box_s_max = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, max_s32} : `VX_CFG_XLEN'(max_s32); + + wire [`VX_CFG_XLEN-1:0] box_h_res = {{(`VX_CFG_XLEN-16){1'b1}}, fp_16_res}; + wire [`VX_CFG_XLEN-1:0] box_h_qnan = {{(`VX_CFG_XLEN-16){1'b1}}, qnan_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_inf = {{(`VX_CFG_XLEN-16){1'b1}}, inf_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_zero = {{(`VX_CFG_XLEN-16){1'b1}}, zero_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_max = {{(`VX_CFG_XLEN-16){1'b1}}, max_h16}; + + wire [`VX_CFG_XLEN-1:0] normal_dst = dst_d_s5 ? fp64_boxed : dst_h_s5 ? box_h_res : f32_boxed; - // F2F special-case results. wire is_nan_src = is_f2f_s5 & fclass_s5.is_nan; wire is_inf_src = is_f2f_s5 & fclass_s5.is_inf; wire is_zero_src = is_f2f_s5 & fclass_s5.is_zero; + wire is_fin_src = is_f2f_s5 & ~fclass_s5.is_nan & ~fclass_s5.is_inf & ~fclass_s5.is_zero; + + wire of_to_inf = (frm_s5 == INST_FRM_RNE) || (frm_s5 == INST_FRM_RMM) + || ((frm_s5 == INST_FRM_RUP) && ~input_sign_s5) + || ((frm_s5 == INST_FRM_RDN) && input_sign_s5); + wire of_post = is_fin_src & ~dst_d_s5 & ~f2f_of_s5 + & (dst_h_s5 ? (fp16_exp == 5'h1F) : (fp32_exp == 8'hFF)); + + wire [1:0] fine_rounded; + VX_fp_rounding #( + .DAT_WIDTH (2) + ) fine_rounding ( + .abs_value_i (2'b01), + .sign_i (input_sign_s5), + .round_sticky_bits_i (fine_round_s5[1:0]), + .rnd_mode_i (frm_s5), + .effective_subtraction_i (1'b0), + .abs_rounded_o (fine_rounded), + `UNUSED_PIN (sign_o), + `UNUSED_PIN (exact_zero_o) + ); + `UNUSED_VAR (fine_rounded[0]) + wire not_tiny = f2f_tiny0_s5 & fine_round_s5[2] & fine_rounded[1]; reg [`VX_CFG_XLEN-1:0] dst_r; always_comb begin if (is_nan_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_qnan_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_qnan_32} : `VX_CFG_XLEN'(f2f_qnan_32)); + dst_r = dst_d_s5 ? qnan_d : dst_h_s5 ? box_h_qnan : box_s_qnan; else if (is_inf_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_inf_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_inf_32} : `VX_CFG_XLEN'(f2f_inf_32)); + dst_r = dst_d_s5 ? inf_d : dst_h_s5 ? box_h_inf : box_s_inf; else if (is_zero_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_zero_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, input_sign_s5, 31'd0} : `VX_CFG_XLEN'({input_sign_s5, 31'd0})); - else if (f2f_of_s5) // FCVT.S.D overflow -> F32 inf - dst_r = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_inf_32} : `VX_CFG_XLEN'(f2f_inf_32); + dst_r = dst_d_s5 ? zero_d : dst_h_s5 ? box_h_zero : box_s_zero; + else if (f2f_of_s5) + dst_r = dst_h_s5 ? (of_to_inf ? box_h_inf : box_h_max) + : (of_to_inf ? box_s_inf : box_s_max); else dst_r = normal_dst; end - assign fp_dst_res = dst_r; - assign f2f_nv_s5 = is_f2f_s5 & fclass_s5.is_signaling; - assign f2f_nx_s5 = is_f2f_s5 & ~fclass_s5.is_nan & ~fclass_s5.is_inf & ~fclass_s5.is_zero & (|round_sticky_bits_s5); + assign fp_dst_res = dst_r; + assign f2f_nv_s5 = is_f2f_s5 & fclass_s5.is_signaling; + assign f2f_of_out_s5 = f2f_of_s5 | of_post; + assign f2f_uf_out_s5 = f2f_uf_s5 & ~not_tiny; + assign f2f_nx_s5 = is_fin_src & ((|round_sticky_bits_s5) | f2f_of_out_s5); end else begin : g_fp_pack_s - assign fp_dst_res = f32_boxed; - assign f2f_nv_s5 = 1'b0; - assign f2f_nx_s5 = 1'b0; + `UNUSED_VAR ({dst_h_s5, f2f_tiny0_s5, fine_round_s5}) + assign fp_dst_res = f32_boxed; + assign f2f_nv_s5 = 1'b0; + assign f2f_nx_s5 = 1'b0; + assign f2f_of_out_s5 = 1'b0; + assign f2f_uf_out_s5 = 1'b0; end // F2I saturation values for NaN/Inf inputs @@ -578,10 +668,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; res_val_64 = 64'(fp_dst_res); res_val_32 = fp_dst_res[31:0]; final_fflags_s5.NV = f2f_nv_s5; - final_fflags_s5.OF = f2f_of_s5; - final_fflags_s5.UF = f2f_uf_s5 & (|round_sticky_bits_s5); - final_fflags_s5.NX = is_f2f_s5 ? (f2f_nx_s5 | f2f_of_s5 | (f2f_uf_s5 & (|round_sticky_bits_s5))) - : (|round_sticky_bits_s5); + final_fflags_s5.OF = f2f_of_out_s5; + final_fflags_s5.UF = f2f_uf_out_s5 & (|round_sticky_bits_s5); + final_fflags_s5.NX = is_f2f_s5 ? f2f_nx_s5 : (|round_sticky_bits_s5); end end diff --git a/hw/rtl/fpu/VX_fncp_unit.sv b/hw/rtl/fpu/VX_fncp_unit.sv index 2526ddafbe..1cdd57db72 100644 --- a/hw/rtl/fpu/VX_fncp_unit.sv +++ b/hw/rtl/fpu/VX_fncp_unit.sv @@ -47,6 +47,11 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `UNUSED_VAR (fmt) wire is_d = HAS_D ? fmt[0] : 1'b0; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire is_h = fmt[1] && (op_type != INST_FPU_CMP) && (frm[2:1] == 2'b10); +`else + wire is_h = 1'b0; +`endif // F32 NaN-boxing (RISC-V spec): on a FLEN>32 register file a single-precision // operand is valid only if the upper bits are all ones; otherwise the operand @@ -149,6 +154,7 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( fclass_t a_fclass_s0, b_fclass_s0; wire a_smaller_s0, ab_equal_s0; wire is_d_s0; + wire is_h_s0; wire a32_boxed_s0; wire [3:0] op_mod = {(op_type == INST_FPU_CMP), frm}; @@ -158,14 +164,14 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( // a non-NaN-boxed single-precision source. Sign/class/ordering are already // derived above from the NaN-box-checked operands. VX_pipe_register #( - .DATAW (4 + 2 * FLEN + 1 + 1 + 2 * $bits(fclass_t) + 1 + 1 + 1 + 1), + .DATAW (4 + 2 * FLEN + 1 + 1 + 2 * $bits(fclass_t) + 1 + 1 + 1 + 1 + 1), .DEPTH (LATENCY > 0) ) pipe_reg0 ( .clk (clk), .reset (reset), .enable (enable && mask), - .data_in ({op_mod, dataa, datab, a_sign_sel, b_sign_sel, a_class_sel, b_class_sel, a_smaller_sel, ab_equal_sel, is_d, a32_boxed}), - .data_out ({op_mod_s0, dataa_s0, datab_s0, a_sign_s0, b_sign_s0, a_fclass_s0, b_fclass_s0, a_smaller_s0, ab_equal_s0, is_d_s0, a32_boxed_s0}) + .data_in ({op_mod, dataa, datab, a_sign_sel, b_sign_sel, a_class_sel, b_class_sel, a_smaller_sel, ab_equal_sel, is_d, is_h, a32_boxed}), + .data_out ({op_mod_s0, dataa_s0, datab_s0, a_sign_s0, b_sign_s0, a_fclass_s0, b_fclass_s0, a_smaller_s0, ab_equal_s0, is_d_s0, is_h_s0, a32_boxed_s0}) ); `UNUSED_VAR (b_fclass_s0) @@ -324,6 +330,16 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( assign result_xlen_s0 = result_s0; end + wire [`VX_CFG_XLEN-1:0] result_out_s0; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire [`VX_CFG_XLEN-1:0] result_h_s0 = op_mod_s0[0] ? {{(`VX_CFG_XLEN-16){1'b1}}, dataa_s0[15:0]} + : {{(`VX_CFG_XLEN-16){dataa_s0[15]}}, dataa_s0[15:0]}; + assign result_out_s0 = is_h_s0 ? result_h_s0 : result_xlen_s0; +`else + `UNUSED_VAR (is_h_s0) + assign result_out_s0 = result_xlen_s0; +`endif + wire fflags_NV; VX_pipe_register #( @@ -333,7 +349,7 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-2]), - .data_in ({result_xlen_s0, fflags_NV_s0}), + .data_in ({result_out_s0, fflags_NV_s0}), .data_out ({result, fflags_NV}) ); // NV, DZ, OF, UF, NX diff --git a/hw/rtl/fpu/VX_fpu_dpi.sv b/hw/rtl/fpu/VX_fpu_dpi.sv index 605e1a48a3..411751225a 100644 --- a/hw/rtl/fpu/VX_fpu_dpi.sv +++ b/hw/rtl/fpu/VX_fpu_dpi.sv @@ -77,7 +77,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( reg [FPC_BITS-1:0] core_select; reg is_fadd, is_fsub, is_fmul, is_fmadd, is_fmsub, is_fnmadd, is_fnmsub; - reg is_div, is_fcmp, is_itof, is_utof, is_ftoi, is_ftou, is_f2f; + reg is_div, is_fcmp, is_itof, is_utof, is_ftoi, is_ftou, is_f2f, is_f2h; reg [NUM_LANES-1:0][63:0] operands [3]; @@ -107,6 +107,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( is_ftoi = 0; is_ftou = 0; is_f2f = 0; + is_f2h = 0; case (op_type) INST_FPU_ADD: begin core_select = FPU_FMA; is_fadd = ~i_fmt; is_fsub = i_fmt; end @@ -121,6 +122,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( INST_FPU_I2F: begin core_select = FPU_CVT; is_itof = 1; end INST_FPU_U2F: begin core_select = FPU_CVT; is_utof = 1; end INST_FPU_F2F: begin core_select = FPU_CVT; is_f2f = 1; end + INST_FPU_F2H: begin core_select = FPU_CVT; is_f2h = 1; end default: begin core_select = FPU_NCP; end endcase end @@ -293,26 +295,29 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire fcvt_ready = per_core_ready_out[FPU_CVT] || ~per_core_valid_out[FPU_CVT]; wire fcvt_fire = fcvt_valid && fcvt_ready; + wire [1:0] f2f_dst_fmt = is_f2h ? 2'd2 : {1'b0, f_fmt}; + wire [1:0] f2f_src_fmt = is_f2h ? {1'b0, f_fmt} : (i_fmt ? 2'd2 : {1'b0, ~f_fmt}); + always @(*) begin for (integer i = 0; i < NUM_LANES; ++i) begin dpi_itof (fcvt_fire, int'(f_fmt), int'(i_fmt), operands[0][i], frm, result_itof[i], fflags_itof[i]); dpi_utof (fcvt_fire, int'(f_fmt), int'(i_fmt), operands[0][i], frm, result_utof[i], fflags_utof[i]); dpi_ftoi (fcvt_fire, int'(i_fmt), int'(f_fmt), operands[0][i], frm, result_ftoi[i], fflags_ftoi[i]); dpi_ftou (fcvt_fire, int'(i_fmt), int'(f_fmt), operands[0][i], frm, result_ftou[i], fflags_ftou[i]); - dpi_f2f (fcvt_fire, int'(f_fmt), int'($bits(f_fmt)'(~f_fmt)), operands[0][i], frm, result_f2f[i], fflags_f2f[i]); + dpi_f2f (fcvt_fire, int'(f2f_dst_fmt), int'(f2f_src_fmt), operands[0][i], frm, result_f2f[i], fflags_f2f[i]); result_fcvt[i] = is_itof ? result_itof[i][`VX_CFG_XLEN-1:0] : is_utof ? result_utof[i][`VX_CFG_XLEN-1:0] : is_ftoi ? result_ftoi[i][`VX_CFG_XLEN-1:0] : is_ftou ? result_ftou[i][`VX_CFG_XLEN-1:0] : - is_f2f ? result_f2f[i][`VX_CFG_XLEN-1:0] : + (is_f2f || is_f2h) ? result_f2f[i][`VX_CFG_XLEN-1:0] : '0; fflags_fcvt[i] = is_itof ? fflags_itof[i] : is_utof ? fflags_utof[i] : is_ftoi ? fflags_ftoi[i] : is_ftou ? fflags_ftou[i] : - is_f2f ? fflags_f2f[i] : + (is_f2f || is_f2h) ? fflags_f2f[i] : '0; end end @@ -376,6 +381,12 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( dpi_fsgnjx (fncp_fire, int'(f_fmt), operands[0][i], operands[1][i], result_fsgnjx[i]); result_fmvx[i] = f_fmt ? operands[0][i] : 64'($signed(operands[0][i][31:0])); // sign-extension result_fmvf[i] = f_fmt ? operands[0][i] : (operands[0][i] | 64'hffffffff00000000); // nan-boxing + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (i_fmt) begin + result_fmvx[i] = 64'($signed(operands[0][i][15:0])); + result_fmvf[i] = operands[0][i] | 64'hffffffffffff0000; + end + `endif end end diff --git a/hw/rtl/fpu/VX_fpu_dsp.sv b/hw/rtl/fpu/VX_fpu_dsp.sv index 240e52cd59..d1a080cc4d 100644 --- a/hw/rtl/fpu/VX_fpu_dsp.sv +++ b/hw/rtl/fpu/VX_fpu_dsp.sv @@ -96,7 +96,7 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_FPCORES-1:0] per_core_ready_out; // Decode fpu core type - wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F) ? FPCORES_BITS'(FPU_CVT) + wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F || op_type == INST_FPU_F2H) ? FPCORES_BITS'(FPU_CVT) : FPCORES_BITS'(op_type[3:2]); VX_stream_switch #( @@ -590,17 +590,21 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [INST_FMT_BITS-1:0] pe_fmt = pe_shared[INST_FRM_BITS+:INST_FMT_BITS]; wire [INST_FPU_BITS-1:0] pe_op = pe_shared[INST_FRM_BITS+INST_FMT_BITS+:INST_FPU_BITS]; - wire is_f2f = (pe_op == INST_FPU_F2F); + wire is_f2h = (pe_op == INST_FPU_F2H); + wire is_f2f = (pe_op == INST_FPU_F2F) || is_f2h; wire is_itof = pe_op[1] & ~is_f2f; wire is_ftoi = ~pe_op[1] & ~is_f2f; wire is_signed = ~pe_op[0]; wire is_int64 = pe_fmt[1]; - wire src_fmt = is_f2f ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format + wire src_fmt = (is_f2f && !is_f2h) ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format wire dst_fmt = pe_fmt[0]; VX_fcvt_unit #( .LATENCY (CVT_LATENCY), .FLEN (`VX_CFG_FLEN), + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + .HALF_ENABLE (1), + `endif .OUT_REG (1) ) fcvt_unit ( .clk (clk), @@ -615,6 +619,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .is_int64 (is_int64), .src_fmt (src_fmt), .dst_fmt (dst_fmt), + .src_h (is_f2f && !is_f2h && pe_fmt[1]), + .dst_h (is_f2h), .dataa (pe_data_in[i][0+:`VX_CFG_XLEN]), .result (pe_data_out[i][0+:`VX_CFG_XLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/hw/rtl/fpu/VX_fpu_fpnew.sv b/hw/rtl/fpu/VX_fpu_fpnew.sv index 22ddf4f347..8ce3b17b72 100644 --- a/hw/rtl/fpu/VX_fpu_fpnew.sv +++ b/hw/rtl/fpu/VX_fpu_fpnew.sv @@ -56,20 +56,26 @@ module VX_fpu_fpnew localparam LATENCY_FDIVSQRT = `MAX(`VX_CFG_FDIV_LATENCY, `VX_CFG_FSQRT_LATENCY); localparam RSP_DATAW = (NUM_LANES * `VX_CFG_XLEN) + 1 + $bits(fflags_t) + TAG_WIDTH; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + localparam [4:0] FP16_FMT_MASK = 5'b00100; +`else + localparam [4:0] FP16_FMT_MASK = 5'b00000; +`endif + localparam fpnew_pkg::fpu_features_t FPU_FEATURES = '{ Width: unsigned'(`VX_CFG_XLEN), EnableVectors: 1'b0, `ifdef VX_CFG_XLEN_64 EnableNanBox: 1'b1, `ifdef VX_CFG_FLEN_64 - FpFmtMask: 5'b11000, + FpFmtMask: 5'b11000 | FP16_FMT_MASK, `else - FpFmtMask: 5'b11000, // FP32+FP64 enabled to avoid CVT bug in FpNew + FpFmtMask: 5'b11000 | FP16_FMT_MASK, // FP32+FP64 enabled to avoid CVT bug in FpNew `endif IntFmtMask: 4'b0011 `else - EnableNanBox: 1'b0, - FpFmtMask: 5'b10000, + EnableNanBox: (FP16_FMT_MASK != 0), + FpFmtMask: 5'b10000 | FP16_FMT_MASK, IntFmtMask: 4'b0010 `endif }; @@ -142,7 +148,25 @@ module VX_fpu_fpnew INST_FPU_NMADD: begin fpu_op = fpnew_pkg::FNMSUB; fpu_op_mod = ~fmt[1]; end INST_FPU_DIV: begin fpu_op = fpnew_pkg::DIV; end INST_FPU_SQRT: begin fpu_op = fpnew_pkg::SQRT; end - `ifdef VX_CFG_FLEN_64 + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + INST_FPU_F2F: begin + fpu_op = fpnew_pkg::F2F; + `ifdef VX_CFG_FLEN_64 + fpu_src_fmt = fmt[1] ? fpnew_pkg::FP16 : (fmt[0] ? fpnew_pkg::FP32 : fpnew_pkg::FP64); + `else + fpu_src_fmt = fpnew_pkg::FP16; + `endif + end + INST_FPU_F2H: begin + fpu_op = fpnew_pkg::F2F; + fpu_dst_fmt = fpnew_pkg::FP16; + `ifdef VX_CFG_FLEN_64 + fpu_src_fmt = fmt[0] ? fpnew_pkg::FP64 : fpnew_pkg::FP32; + `else + fpu_src_fmt = fpnew_pkg::FP32; + `endif + end + `elsif VX_CFG_FLEN_64 INST_FPU_F2F: begin fpu_op = fpnew_pkg::F2F; fpu_src_fmt = fmt[0] ? fpnew_pkg::FP32 : fpnew_pkg::FP64; end `endif INST_FPU_F2I, @@ -154,7 +178,15 @@ module VX_fpu_fpnew case (frm) 0,1,2: begin fpu_op = fpnew_pkg::SGNJ; fpu_rnd = {1'b0, frm[1:0]}; fpu_has_fflags = 0; end // FSGNJ 3: begin fpu_op = fpnew_pkg::CLASSIFY; fpu_has_fflags = 0; end // CLASS - 4,5: begin fpu_op = fpnew_pkg::SGNJ; fpu_rnd = 3'b011; fpu_op_mod = ~frm[0]; fpu_has_fflags = 0; end // FMV.X.W, FMV.W.X + 4,5: begin + fpu_op = fpnew_pkg::SGNJ; fpu_rnd = 3'b011; fpu_op_mod = ~frm[0]; fpu_has_fflags = 0; + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (fmt[1]) begin + fpu_src_fmt = fpnew_pkg::FP16; + fpu_dst_fmt = fpnew_pkg::FP16; + end + `endif + end 6,7: begin fpu_op = fpnew_pkg::MINMAX; fpu_rnd = {2'b00, frm[0]}; end // MIN, MAX endcase end diff --git a/hw/rtl/fpu/VX_fpu_std.sv b/hw/rtl/fpu/VX_fpu_std.sv index 5fed9717f4..81a24a8e52 100644 --- a/hw/rtl/fpu/VX_fpu_std.sv +++ b/hw/rtl/fpu/VX_fpu_std.sv @@ -87,7 +87,7 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_FPCORES-1:0] per_core_ready_out; // Decode FPU core selector - wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F) ? FPCORES_BITS'(FPU_CVT) + wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F || op_type == INST_FPU_F2H) ? FPCORES_BITS'(FPU_CVT) : FPCORES_BITS'(op_type[3:2]); VX_stream_switch #( @@ -413,17 +413,21 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [INST_FMT_BITS-1:0] pe_fmt = pe_shared[INST_FRM_BITS+:INST_FMT_BITS]; wire [INST_FPU_BITS-1:0] pe_op = pe_shared[INST_FRM_BITS+INST_FMT_BITS+:INST_FPU_BITS]; - wire is_f2f = (pe_op == INST_FPU_F2F); + wire is_f2h = (pe_op == INST_FPU_F2H); + wire is_f2f = (pe_op == INST_FPU_F2F) || is_f2h; wire is_itof = pe_op[1] & ~is_f2f; wire is_ftoi = ~pe_op[1] & ~is_f2f; wire is_signed = ~pe_op[0]; wire is_int64 = pe_fmt[1]; - wire src_fmt = is_f2f ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format + wire src_fmt = (is_f2f && !is_f2h) ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format wire dst_fmt = pe_fmt[0]; VX_fcvt_unit #( .LATENCY (CVT_LATENCY), .FLEN (`VX_CFG_FLEN), + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + .HALF_ENABLE (1), + `endif .OUT_REG (1) ) fcvt_unit ( .clk (clk), @@ -438,6 +442,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .is_int64 (is_int64), .src_fmt (src_fmt), .dst_fmt (dst_fmt), + .src_h (is_f2f && !is_f2h && pe_fmt[1]), + .dst_h (is_f2h), .dataa (pe_data_in[i][0+:`VX_CFG_XLEN]), .result (pe_data_out[i][0+:`VX_CFG_XLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/hw/unittest/fcvt_unit/Makefile b/hw/unittest/fcvt_unit/Makefile index 7e5bf4d7d7..fae0143d34 100644 --- a/hw/unittest/fcvt_unit/Makefile +++ b/hw/unittest/fcvt_unit/Makefile @@ -24,6 +24,12 @@ PARAMS += -GSUBNORM_ENABLE=0 -GEXCEPT_ENABLE=0 CXXFLAGS += -DLEAN endif +HALF ?= 1 +ifeq ($(HALF),1) +PARAMS += -GHALF_ENABLE=1 +CXXFLAGS += -DHALF +endif + SRCS += $(SW_COMMON_DIR)/softfloat_ext.cpp SRCS += $(SW_COMMON_DIR)/rvfloats.cpp SRCS += $(SRC_DIR)/main.cpp diff --git a/hw/unittest/fcvt_unit/main.cpp b/hw/unittest/fcvt_unit/main.cpp index 1a7e5074ee..d9b21d8f62 100644 --- a/hw/unittest/fcvt_unit/main.cpp +++ b/hw/unittest/fcvt_unit/main.cpp @@ -12,8 +12,9 @@ // limitations under the License. // Functional test for VX_fcvt_unit (XLEN=32 / F32): exercises I2F and F2I in -// both signed and unsigned forms and checks bit-exact result/fflags against the -// softfloat (rvfloats) RISC-V golden across all rounding modes. Strict check. +// both signed and unsigned forms, plus F32<->F16 with HALF, and checks +// bit-exact result/fflags against the softfloat (rvfloats) RISC-V golden across +// all rounding modes. Strict check. #include "VVX_fcvt_unit.h" #include "vl_simulator.h" @@ -32,11 +33,15 @@ using Device = VVX_fcvt_unit; -enum { I2F_S, I2F_U, F2I_S, F2I_U, NUM_OPS }; +#ifdef HALF +enum { I2F_S, I2F_U, F2I_S, F2I_U, F2H, H2F, NUM_OPS }; +#else +enum { I2F_S, I2F_U, F2I_S, F2I_U, NUM_OPS, F2H, H2F }; +#endif namespace { -const char* op_name[NUM_OPS] = {"i2f_s","i2f_u","f2i_s","f2i_u"}; +const char* op_name[] = {"i2f_s","i2f_u","f2i_s","f2i_u","f2h","h2f"}; struct TV { uint32_t a, frm; int op; }; struct Golden { uint32_t res, fflags, a, frm; int op; }; @@ -48,8 +53,16 @@ const uint32_t kSpecials[] = { 0x7f800000, 0xff800000, 0x7fc00000, 0x4f000000, 0xcf000000, 0x4effffff, 0x00000001, 0xffffffff, 0x7fffffff, 0x00000002, 0x000003e8, 0x40490fdb, 0x5f000000, 0x5effffff, 0x3f000000, 0x4c3b0244, + 0x477fe000, 0x477fefff, 0x477ff000, 0x477ff001, 0x47800000, 0xc77ff000, + 0x38800000, 0x387fffff, 0x387ff000, 0x387fe000, 0x33800000, 0x33000000, + 0x33000001, 0x32ffffff, 0x3f801000, 0x3f803000, 0x7f800001, 0x7f7fffff, + 0xffff7c00, 0xffff7c01, 0xffff7e00, 0xffff0001, 0xffff83ff, 0xffff3c00, + 0x00003c00, 0x7fff7bff, }; +inline uint32_t flush_sub(uint32_t x){ return (((x>>23)&0xFF)==0 && (x&0x7FFFFF)) ? (x&0x80000000u) : x; } +inline uint16_t flush_sub16(uint16_t x){ return (((x>>10)&0x1F)==0 && (x&0x3FF)) ? (x&0x8000u) : x; } + uint32_t golden(int op, uint32_t a, uint32_t frm, uint32_t* ff) { *ff = 0; switch (op) { @@ -57,6 +70,8 @@ uint32_t golden(int op, uint32_t a, uint32_t frm, uint32_t* ff) { case I2F_U: return rv_utof_s(a, frm, ff); case F2I_S: return rv_ftoi_s(a, frm, ff); case F2I_U: return rv_ftou_s(a, frm, ff); + case F2H: return 0xffff0000u | rv_ftoh_s(a, frm, ff); + case H2F: return ((a >> 16) == 0xffff) ? rv_htof_s(uint16_t(a), frm, ff) : 0x7fc00000u; } return 0; } @@ -65,17 +80,19 @@ void drive(Device& dut, int op, uint32_t a, uint32_t frm) { dut.frm = frm; dut.is_itof = (op == I2F_S || op == I2F_U); dut.is_ftoi = (op == F2I_S || op == F2I_U); - dut.is_f2f = 0; + dut.is_f2f = (op == F2H || op == H2F); dut.is_signed = (op == I2F_S || op == F2I_S); dut.is_int64 = 0; // XLEN=32 dut.src_fmt = 0; // F32 dut.dst_fmt = 0; // F32 +#ifdef HALF + dut.src_h = (op == H2F); + dut.dst_h = (op == F2H); +#endif dut.dataa = a; dut.mask = 1; } -inline uint32_t flush_sub(uint32_t x){ return (((x>>23)&0xFF)==0 && (x&0x7FFFFF)) ? (x&0x80000000u) : x; } - } // namespace int main(int argc, char** argv) { @@ -93,8 +110,12 @@ int main(int argc, char** argv) { for (uint32_t a : kSpecials) for (uint32_t f = 0; f < 5; ++f) tests.push_back({a, f, op}); - for (size_t i = 0; i < num_random; ++i) - tests.push_back({static_cast(rng()), static_cast(i % 5), op}); + for (size_t i = 0; i < num_random; ++i) { + uint32_t a = static_cast(rng()); + if (op == H2F && (i % 16) != 0) a |= 0xffff0000u; + if (op == F2H && (i % 2) == 0) a = (a & 0x807fffffu) | ((97u + (a >> 24) % 50u) << 23); + tests.push_back({a, static_cast(i % 5), op}); + } } const size_t N = tests.size(); @@ -117,7 +138,8 @@ int main(int argc, char** argv) { uint32_t ff = 0; uint32_t ga = t.a; #ifdef LEAN - if (t.op == F2I_S || t.op == F2I_U) ga = flush_sub(t.a); // DAZ float source + if (t.op == F2I_S || t.op == F2I_U || t.op == F2H) ga = flush_sub(t.a); // DAZ float source + if (t.op == H2F && (t.a >> 16) == 0xffff) ga = 0xffff0000u | flush_sub16(uint16_t(t.a)); #endif uint32_t r = golden(t.op, ga, t.frm, &ff); #ifdef LEAN @@ -146,7 +168,7 @@ int main(int argc, char** argv) { } printf("fcvt_unit: checked=%zu errors=%zu per-op:", checked, errors); - for (int op = 0; op < NUM_OPS; ++op) printf(" %s=%zu", op_name[op], per_op[op]); + for (int op = 0; op < (int)NUM_OPS; ++op) printf(" %s=%zu", op_name[op], per_op[op]); printf("\n%s\n", errors ? "FAILED" : "PASSED"); return errors ? 1 : 0; } diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 248d89c8b2..7fe704d961 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -219,7 +219,11 @@ static op_string_t op_string(const Instr &instr) { std::abort(); } } - case FpuType::F2F: return {fpuArgs.is_f64 ? "FCVT.D.S":"FCVT.S.D", ""}; + case FpuType::F2F: { + if (fpuArgs.is_f16) return {(fpuArgs.cvt == 1) ? "FCVT.H.D":"FCVT.H.S", ""}; + if (fpuArgs.cvt == 2) return {fpuArgs.is_f64 ? "FCVT.D.H":"FCVT.S.H", ""}; + return {fpuArgs.is_f64 ? "FCVT.D.S":"FCVT.S.D", ""}; + } case FpuType::FCMP: { switch (fpuArgs.frm) { case 0: return {fpuArgs.is_f64 ? "FLE.D":"FLE.S", ""}; @@ -239,8 +243,8 @@ static op_string_t op_string(const Instr &instr) { } } case FpuType::FCLASS: return {fpuArgs.is_f64 ? "FCLASS.D":"FCLASS.S", ""}; - case FpuType::FMVXW: return {fpuArgs.is_f64 ? "FMV.X.D":"FMV.X.S", ""}; - case FpuType::FMVWX: return {fpuArgs.is_f64 ? "FMV.D.X":"FMV.S.X", ""}; + case FpuType::FMVXW: return {fpuArgs.is_f16 ? "FMV.X.H" : fpuArgs.is_f64 ? "FMV.X.D":"FMV.X.S", ""}; + case FpuType::FMVWX: return {fpuArgs.is_f16 ? "FMV.H.X" : fpuArgs.is_f64 ? "FMV.D.X":"FMV.S.X", ""}; case FpuType::FMINMAX: { switch (fpuArgs.frm) { case 0: return {fpuArgs.is_f64 ? "FMIN.D":"FMIN.S", ""}; @@ -258,11 +262,11 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); - // packLD detection: Float dest with sub-word width (LB or LH). - if (dst_float && lsuArgs.width == 0) return {"PACKLB.F", ""}; - if (dst_float && lsuArgs.width == 1) return {"PACKLH.F", ""}; + if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) return {"PACKLB.F", ""}; + if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) return {"PACKLH.F", ""}; if (dst_float) { switch (lsuArgs.width) { + case 1: return {"FLH", to_hex_str(lsuArgs.offset)}; case 2: return {"FLW", to_hex_str(lsuArgs.offset)}; case 3: return {"FLD", to_hex_str(lsuArgs.offset)}; default: @@ -288,6 +292,7 @@ static op_string_t op_string(const Instr &instr) { bool src_float = (instr.get_src_reg(1).type == RegType::Float); if (src_float) { switch (lsuArgs.width) { + case 1: return {"FSH", to_hex_str(lsuArgs.offset)}; case 2: return {"FSW", to_hex_str(lsuArgs.offset)}; case 3: return {"FSD", to_hex_str(lsuArgs.offset)}; default: @@ -710,7 +715,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } break; case Opcode::FCI: { instr->set_fu_type(FUType::FPU); - instr->set_args(IntrFpuArgs{funct3, rs2, (funct7 & 0x1)}); + instr->set_args(IntrFpuArgs{funct3, rs2, (funct7 & 0x1), ((funct7 & 0x3) == 0x2)}); // Most FP ops raise exception flags; a dynamic rounding mode reads frm. uint8_t frm_rd = (funct3 == 0x7) ? Instr::FCSR_FRM : 0; switch (funct7) { @@ -761,13 +766,37 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_src_reg(1, rs2, RegType::Float); instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); break; - case 0x20: // FCVT.S.D - case 0x21: // FCVT.D.S + case 0x20: // FCVT.S.D, FCVT.S.H + case 0x21: // FCVT.D.S, FCVT.D.H +#ifndef VX_CFG_EXT_ZFHMIN_ENABLE + if ((rs2 & 0x3) == 0x2) + std::abort(); +#endif instr->set_op_type(FpuType::F2F); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); break; +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE + case 0x22: // FCVT.H.S, FCVT.H.D + instr->set_op_type(FpuType::F2F); + instr->set_dest_reg(rd, RegType::Float); + instr->set_src_reg(0, rs1, RegType::Float); + instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); + break; + case 0x72: // FMV.X.H + if (funct3 != 0) + std::abort(); + instr->set_op_type(FpuType::FMVXW); + instr->set_dest_reg(rd, RegType::Integer); + instr->set_src_reg(0, rs1, RegType::Float); + break; + case 0x7a: // FMV.H.X + instr->set_op_type(FpuType::FMVWX); + instr->set_dest_reg(rd, RegType::Float); + instr->set_src_reg(0, rs1, RegType::Integer); + break; +#endif case 0x2c: // FSQRT.S case 0x2d: // FSQRT.D instr->set_op_type(FpuType::FSQRT); @@ -823,7 +852,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_op_type((op == Opcode::FMADD) ? FpuType::FMADD : (op == Opcode::FMSUB) ? FpuType::FMSUB : (op == Opcode::FNMADD) ? FpuType::FNMADD : FpuType::FNMSUB); - instr->set_args(IntrFpuArgs{funct3, funct2, (funct7 & 0x1)}); + instr->set_args(IntrFpuArgs{funct3, funct2, (funct7 & 0x1), 0}); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); instr->set_src_reg(1, rs2, RegType::Float); diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 270f7b8a3f..0e29004275 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -37,6 +37,16 @@ inline int64_t check_boxing(int64_t a) { return a; return nan_box(0x7fc00000); // NaN } +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE +inline uint64_t nan_box16(uint16_t value) { + return value | 0xffffffffffff0000; +} +inline uint16_t check_boxing16(uint64_t a) { + if ((a >> 16) == 0xffffffffffff) + return uint16_t(a); + return 0x7e00; +} +#endif } FpuUnit::FpuUnit(const SimContext& ctx, const char* name, Core* core) @@ -271,6 +281,22 @@ void FpuUnit::execute(instr_trace_t* trace) { if (!tmask.test(t)) continue; uint32_t frm = emu.get_fpu_rm(fpuArgs.frm, wid, t); uint32_t fflags = 0; +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (fpuArgs.is_f16) { + if (fpuArgs.cvt == 1) { + rd_data[t].u64 = nan_box16(rv_dtoh(rs1_data[t].u64, frm, &fflags)); + } else { + rd_data[t].u64 = nan_box16(rv_ftoh_s(check_boxing(rs1_data[t].u64), frm, &fflags)); + } + } else if (fpuArgs.cvt == 2) { + auto h = check_boxing16(rs1_data[t].u64); + if (fpuArgs.is_f64) { + rd_data[t].u64 = rv_htod(h, frm, &fflags); + } else { + rd_data[t].u64 = nan_box(rv_htof_s(h, frm, &fflags)); + } + } else +#endif if (fpuArgs.is_f64) { rd_data[t].u64 = rv_ftod(check_boxing(rs1_data[t].u64), frm, &fflags); } else { @@ -295,6 +321,11 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (fpuArgs.is_f16) { + rd_data[t].i = sext((uint64_t)uint16_t(rs1_data[t].u64), 16); + } else +#endif if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].u64; } else { @@ -308,6 +339,11 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (fpuArgs.is_f16) { + rd_data[t].u64 = nan_box16(uint16_t(rs1_data[t].i)); + } else +#endif if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].i; } else { diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 78fd3fd466..88e8f63207 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -266,9 +266,12 @@ void LsuUnit::process_response_step(uint32_t b) { uint64_t formatted = 0; switch (entry.lsu_args.width) { case 0: // LB - case 1: // LH formatted = (uint64_t)(int64_t)sext((Word)read_data, data_width); break; + case 1: // LH (FLH: raw bits for Float dest, NaN-boxed below) + formatted = dst_float ? read_data + : (uint64_t)(int64_t)sext((Word)read_data, data_width); + break; case 2: // LW (sign-ext for Integer dest; raw bits for Float dest, NaN-boxed below) formatted = dst_float ? read_data : (uint64_t)(int64_t)sext((Word)read_data, data_width); @@ -284,8 +287,10 @@ void LsuUnit::process_response_step(uint32_t b) { } // Place at bytesel-specified position; OR-in NaN-box for Float dest. auto& dst = entry.trace->dst_data.at(lane_info.tid); - dst.u64 = (formatted << (8 * byte_off)) - | (nan_box ? 0xFFFFFFFF00000000ull : 0); + uint64_t box = !nan_box ? 0 + : (entry.lsu_args.width == 1) ? 0xFFFFFFFFFFFF0000ull + : 0xFFFFFFFF00000000ull; + dst.u64 = (formatted << (8 * byte_off)) | box; } } entry.count -= lsu_rsp.mask.count(); // track remaining diff --git a/sim/simx/types.h b/sim/simx/types.h index cb20410167..63b94bf42c 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -568,6 +568,7 @@ struct IntrFpuArgs { uint32_t frm : 3; uint32_t cvt : 2; uint32_t is_f64 : 1; + uint32_t is_f16 : 1; }; inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { diff --git a/sw/common/rvfloats.cpp b/sw/common/rvfloats.cpp index 11762bccd8..607ae472d7 100644 --- a/sw/common/rvfloats.cpp +++ b/sw/common/rvfloats.cpp @@ -548,6 +548,20 @@ uint16_t rv_ftoh_s(uint32_t a, uint32_t frm, uint32_t* fflags) { return from_float16_t(r); } +uint64_t rv_htod(uint16_t a, uint32_t frm, uint32_t* fflags) { + rv_init(frm); + auto r = f16_to_f64(to_float16_t(a)); + if (fflags) { *fflags = softfloat_exceptionFlags; } + return from_float64_t(r); +} + +uint16_t rv_dtoh(uint64_t a, uint32_t frm, uint32_t* fflags) { + rv_init(frm); + auto r = f64_to_f16(to_float64_t(a)); + if (fflags) { *fflags = softfloat_exceptionFlags; } + return from_float16_t(r); +} + uint32_t rv_btof_s(uint16_t a, uint32_t frm, uint32_t* fflags) { rv_init(frm); auto r = bf16_to_f32(to_bfloat16_t(a)); diff --git a/sw/common/rvfloats.h b/sw/common/rvfloats.h index 57d03079a5..6b7192b805 100644 --- a/sw/common/rvfloats.h +++ b/sw/common/rvfloats.h @@ -94,6 +94,8 @@ uint64_t rv_ftod(uint32_t a, uint32_t frm, uint32_t* fflags); // fp16 <--> fp32 conversions uint32_t rv_htof_s(uint16_t a, uint32_t frm, uint32_t* fflags); uint16_t rv_ftoh_s(uint32_t a, uint32_t frm, uint32_t* fflags); +uint64_t rv_htod(uint16_t a, uint32_t frm, uint32_t* fflags); +uint16_t rv_dtoh(uint64_t a, uint32_t frm, uint32_t* fflags); // bf16 <--> fp32 conversions uint32_t rv_btof_s(uint16_t a, uint32_t frm, uint32_t* fflags); diff --git a/sw/runtime/include/vortex2.h b/sw/runtime/include/vortex2.h index 7e2e99d6bb..be8916cce3 100644 --- a/sw/runtime/include/vortex2.h +++ b/sw/runtime/include/vortex2.h @@ -110,6 +110,7 @@ typedef struct vx_kernel* vx_kernel_h; #define VX_ISA_EXT_DXA (1ull << (32 + 10)) #define VX_ISA_EXT_RTU (1ull << (32 + 11)) #define VX_ISA_EXT_MUFU (1ull << (32 + 12)) +#define VX_ISA_EXT_ZFHMIN (1ull << (32 + 13)) // ============================================================================ // Device memory access flags (vx_buffer_create / vx_buffer_access) diff --git a/tests/regression/Makefile b/tests/regression/Makefile index 21d7960bfe..e5be8fed12 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax mufu \ + softmax mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/common.mk b/tests/regression/common.mk index ed7a588772..25ffaf4462 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -45,11 +45,17 @@ else ZACAS_EXT := endif +ifneq (,$(filter -DVX_CFG_EXT_ZFHMIN_ENABLE, $(XCONFIGS))) + ZFHMIN_EXT := _zfhmin +else + ZFHMIN_EXT := +endif + ifeq ($(XLEN),64) - VX_CFLAGS += -march=rv64imafd$(C_EXT)$(ZACAS_EXT) -mabi=lp64d + VX_CFLAGS += -march=rv64imafd$(C_EXT)$(ZACAS_EXT)$(ZFHMIN_EXT) -mabi=lp64d STARTUP_ADDR ?= 0x180000000 else - VX_CFLAGS += -march=rv32imaf$(C_EXT)$(ZACAS_EXT) -mabi=ilp32f + VX_CFLAGS += -march=rv32imaf$(C_EXT)$(ZACAS_EXT)$(ZFHMIN_EXT) -mabi=ilp32f STARTUP_ADDR ?= 0x80000000 endif diff --git a/tests/regression/zfhmin/Makefile b/tests/regression/zfhmin/Makefile new file mode 100644 index 0000000000..bba9c53cf3 --- /dev/null +++ b/tests/regression/zfhmin/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_ZFHMIN_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_ZFHMIN_ENABLE) + +PROJECT := zfhmin + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include + +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/zfhmin/common.h b/tests/regression/zfhmin/common.h new file mode 100644 index 0000000000..2dff7131fd --- /dev/null +++ b/tests/regression/zfhmin/common.h @@ -0,0 +1,48 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define ZF_NUM_RM 6 + +enum { + ZF_MODE_NARROW_S = 0, + ZF_MODE_WIDEN_S = 1, + ZF_MODE_NARROW_D = 2, + ZF_MODE_WIDEN_D = 3, + ZF_MODE_FLAGS = 4, + ZF_MODE_BOX = 5, +}; + +enum { + ZF_OP_H_S = 0, + ZF_OP_S_H = 1, + ZF_OP_H_D = 2, + ZF_OP_D_H = 3, +}; + +#define ZF_BOX_OUTS 5 + +typedef struct { + uint32_t op; + uint32_t rm; + uint64_t bits; +} zf_flag_case_t; + +typedef struct { + uint64_t result; + uint32_t fflags; + uint32_t pad; +} zf_flag_out_t; + +typedef struct { + uint32_t mode; + uint32_t count; + uint32_t frm; + uint32_t pad; + uint64_t src_addr; + uint64_t dst_addr; + uint64_t aux_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/zfhmin/kernel.cpp b/tests/regression/zfhmin/kernel.cpp new file mode 100644 index 0000000000..11d14215e5 --- /dev/null +++ b/tests/regression/zfhmin/kernel.cpp @@ -0,0 +1,180 @@ +#include +#include +#include "common.h" + +#if __riscv_xlen == 64 +typedef uint64_t raw_t; +#define FMV_X_RAW "fmv.x.d" +#else +typedef uint32_t raw_t; +#define FMV_X_RAW "fmv.x.w" +#endif + +#define CVT(insn, out, in, rm) __asm__ volatile(insn " %0, %1, " rm : "=f"(out) : "f"(in)) + +static inline __attribute__((always_inline)) void store_h(uint16_t* p, float h) { + __asm__ volatile("fsh %0, 0(%1)" :: "f"(h), "r"(p) : "memory"); +} + +static inline __attribute__((always_inline)) float load_h(const uint16_t* p) { + float h; + __asm__ volatile("flh %0, 0(%1)" : "=f"(h) : "r"(p) : "memory"); + return h; +} + +static inline __attribute__((always_inline)) long mv_x_h(float h) { + long r; + __asm__ volatile("fmv.x.h %0, %1" : "=r"(r) : "f"(h)); + return r; +} + +static inline __attribute__((always_inline)) float mv_h_x(long x) { + float h; + __asm__ volatile("fmv.h.x %0, %1" : "=f"(h) : "r"(x)); + return h; +} + +static inline __attribute__((always_inline)) float mv_w_x(long x) { + float f; + __asm__ volatile("fmv.w.x %0, %1" : "=f"(f) : "r"(x)); + return f; +} + +static inline __attribute__((always_inline)) raw_t mv_x_raw(float f) { + raw_t r; + __asm__ volatile(FMV_X_RAW " %0, %1" : "=r"(r) : "f"(f)); + return r; +} + +static inline __attribute__((always_inline)) uint32_t mv_x_w(float f) { + long r; + __asm__ volatile("fmv.x.w %0, %1" : "=r"(r) : "f"(f)); + return (uint32_t)r; +} + +static void narrow_s(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_NUM_RM; + auto mvx = reinterpret_cast(arg->aux_addr); + float x = src[i]; + float h; + CVT("fcvt.h.s", h, x, "rne"); store_h(dst + 0, h); mvx[i] = mv_x_h(h); + CVT("fcvt.h.s", h, x, "rtz"); store_h(dst + 1, h); + CVT("fcvt.h.s", h, x, "rdn"); store_h(dst + 2, h); + CVT("fcvt.h.s", h, x, "rup"); store_h(dst + 3, h); + CVT("fcvt.h.s", h, x, "rmm"); store_h(dst + 4, h); + CVT("fcvt.h.s", h, x, "dyn"); store_h(dst + 5, h); +} + +static void widen_s(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + float h = load_h(src + i); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(h)); + dst[i] = mv_x_w(y); +} + +#if __riscv_xlen == 64 +static void narrow_d(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_NUM_RM; + double x = src[i]; + float h; + CVT("fcvt.h.d", h, x, "rne"); store_h(dst + 0, h); + CVT("fcvt.h.d", h, x, "rtz"); store_h(dst + 1, h); + CVT("fcvt.h.d", h, x, "rdn"); store_h(dst + 2, h); + CVT("fcvt.h.d", h, x, "rup"); store_h(dst + 3, h); + CVT("fcvt.h.d", h, x, "rmm"); store_h(dst + 4, h); + CVT("fcvt.h.d", h, x, "dyn"); store_h(dst + 5, h); +} + +static void widen_d(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + float h = load_h(src + i); + double y; + __asm__ volatile("fcvt.d.h %0, %1" : "=f"(y) : "f"(h)); + uint64_t r; + __asm__ volatile("fmv.x.d %0, %1" : "=r"(r) : "f"(y)); + dst[i] = r; +} +#endif + +static void flag_case(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + uint32_t op = src[i].op; + uint64_t bits = src[i].bits; + uint64_t res = 0; + long fflags; + __asm__ volatile("fsrm %0" :: "r"(src[i].rm)); + __asm__ volatile("csrw fflags, zero"); + if (op == ZF_OP_H_S) { + float x = mv_w_x((long)(uint32_t)bits); + float h; + CVT("fcvt.h.s", h, x, "dyn"); + res = (uint16_t)mv_x_h(h); + } else if (op == ZF_OP_S_H) { + float h = mv_h_x((long)bits); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(h)); + res = mv_x_w(y); + } +#if __riscv_xlen == 64 + else if (op == ZF_OP_H_D) { + double x; + __asm__ volatile("fmv.d.x %0, %1" : "=f"(x) : "r"(bits)); + float h; + CVT("fcvt.h.d", h, x, "dyn"); + res = (uint16_t)mv_x_h(h); + } else if (op == ZF_OP_D_H) { + float h = mv_h_x((long)bits); + double y; + __asm__ volatile("fcvt.d.h %0, %1" : "=f"(y) : "f"(h)); + __asm__ volatile("fmv.x.d %0, %1" : "=r"(res) : "f"(y)); + } +#endif + __asm__ volatile("csrr %0, fflags" : "=r"(fflags)); + dst[i].result = res; + dst[i].fflags = (uint32_t)fflags; +} + +static void box(const kernel_arg_t* arg, uint32_t i) { + auto hsrc = reinterpret_cast(arg->src_addr); + auto gsrc = reinterpret_cast(arg->aux_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_BOX_OUTS; + long g = (long)gsrc[i]; + dst[0] = (uint64_t)mv_x_raw(load_h(hsrc + i)); + dst[1] = (uint64_t)mv_x_raw(mv_h_x(g)); + dst[2] = (uint64_t)(int64_t)mv_x_h(load_h(hsrc + i)); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(mv_w_x(g))); + dst[3] = mv_x_w(y); + dst[4] = (uint64_t)(int64_t)mv_x_h(mv_w_x(g)); +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (arg->mode == ZF_MODE_FLAGS) { + int nt = vx_num_threads(); + for (int l = 0; l < nt; ++l) { + if (vx_thread_id() == l && i < arg->count) + flag_case(arg, i); + } + return; + } + if (i >= arg->count) + return; + __asm__ volatile("fsrm %0" :: "r"(arg->frm)); + switch (arg->mode) { + case ZF_MODE_NARROW_S: narrow_s(arg, i); break; + case ZF_MODE_WIDEN_S: widen_s(arg, i); break; +#if __riscv_xlen == 64 + case ZF_MODE_NARROW_D: narrow_d(arg, i); break; + case ZF_MODE_WIDEN_D: widen_d(arg, i); break; +#endif + case ZF_MODE_BOX: box(arg, i); break; + default: break; + } +} diff --git a/tests/regression/zfhmin/main.cpp b/tests/regression/zfhmin/main.cpp new file mode 100644 index 0000000000..e35c2cfdb7 --- /dev/null +++ b/tests/regression/zfhmin/main.cpp @@ -0,0 +1,360 @@ +#include +#include +#include "common.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_random = 4096; +uint32_t frm_dyn = 3; +uint32_t phases = 0x3f; +uint32_t verbose = 0; +uint32_t num_flag_in = 160; + +vx_device_h dev = nullptr; +vx_queue_h q = nullptr; +vx_kernel_h kern = nullptr; +uint32_t xlen = 32; +int errors = 0; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:m:p:v:f:k:h")) != -1) { + switch (c) { + case 'n': num_random = std::atoi(optarg); break; + case 'p': phases = std::strtoul(optarg, nullptr, 0); break; + case 'v': verbose = std::atoi(optarg); break; + case 'f': num_flag_in = std::atoi(optarg); break; + case 'm': frm_dyn = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-n random cases] [-m dynamic frm] [-p phase mask] [-v dump count] [-f flag inputs] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +struct Buf { + vx_buffer_h h = nullptr; + uint64_t addr = 0; + uint64_t size = 0; + Buf(uint64_t sz) : size(sz ? sz : 8) { + CHECK(vx_buffer_create(dev, size, VX_MEM_READ_WRITE, &h)); + CHECK(vx_buffer_address(h, &addr)); + } + ~Buf() { vx_buffer_release(h); } + void write(const void* p) { CHECK(vx_enqueue_write(q, h, 0, p, size, 0, nullptr, nullptr)); } +}; + +void run(uint32_t mode, uint32_t count, Buf& src, Buf& dst, Buf* aux, void* out, uint64_t out_size) { + kernel_arg_t arg{}; + arg.mode = mode; + arg.count = count; + arg.frm = frm_dyn; + arg.src_addr = src.addr; + arg.dst_addr = dst.addr; + arg.aux_addr = aux ? aux->addr : 0; + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, out, dst.h, 0, out_size, 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); +} + +void read_aux(Buf& aux, void* out, uint64_t size) { + vx_event_h ev = nullptr; + CHECK(vx_enqueue_read(q, out, aux.h, 0, size, 0, nullptr, &ev)); + CHECK(vx_event_wait_value(ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(ev); +} + +void report(const char* what, uint64_t input, uint32_t rm, uint64_t expected, uint64_t actual) { + if (errors < 32) + std::printf("*** %s in=0x%llx rm=%u expected=0x%llx actual=0x%llx\n", what, + (unsigned long long)input, rm, (unsigned long long)expected, (unsigned long long)actual); + ++errors; +} + +uint32_t rm_of(uint32_t k) { return (k < 5) ? k : frm_dyn; } + +int64_t sext16(uint64_t v) { return (int16_t)(uint16_t)v; } + +std::vector f32_inputs(std::mt19937& rng) { + std::vector v = { + 0x00000000, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, + 0x7f800001, 0xff800001, 0x7fa00000, 0x7fc12345, 0x7f7fffff, 0xff7fffff, + 0x00000001, 0x80000001, 0x007fffff, 0x00800000, + 0x477fe000, 0x477fefff, 0x477ff000, 0x477ff001, 0x477fffff, 0x47800000, + 0xc77fe000, 0xc77ff000, 0xc7800000, + 0x38800000, 0x387fffff, 0x387fe000, 0x387ff000, 0x33800000, 0x33000000, + 0x33000001, 0x32ffffff, 0x33400000, 0x33c00000, 0x2f800000, 0xb3000000, + 0xb3000001, 0x3f800000, 0x3f801000, 0x3f803000, 0x3f801001, 0x3f800fff, + 0xbf801000, 0xbf803000, 0x40490fdb, 0x3eaaaaab, + }; + const uint32_t low[] = {0, 1, 0xfff, 0x1000, 0x1001, 0x1fff, 0x2000, 0x3000, 0x7fffff, 0x7fefff}; + for (int e = -27; e <= 17; ++e) { + for (uint32_t l : low) { + for (int s = 0; s < 2; ++s) { + uint32_t hi = rng() & 0x7fe000; + v.push_back((uint32_t(s) << 31) | (uint32_t(e + 127) << 23) | ((hi | l) & 0x7fffff)); + } + } + } + for (uint32_t i = 0; i < num_random; ++i) { + uint32_t e = 127 - 30 + (rng() % 50); + v.push_back((rng() & 0x807fffff) | (e << 23)); + } + for (uint32_t i = 0; i < num_random / 4; ++i) + v.push_back(rng()); + return v; +} + +std::vector f64_inputs(std::mt19937& rng) { + std::vector v = { + 0x0000000000000000ull, 0x8000000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000000ull, + 0x7ff8000000000000ull, 0x7ff0000000000001ull, 0x7ff4000000000000ull, 0x7fefffffffffffffull, + 0x0000000000000001ull, 0x40effc0000000000ull, 0x40effdffffffffffull, 0x40effe0000000000ull, + 0x40effe0000000001ull, 0x40f0000000000000ull, 0x3f10000000000000ull, 0x3e70000000000000ull, + 0x3e60000000000000ull, 0x3e60000000000001ull, 0x3e5fffffffffffffull, 0x3ff0000000000000ull, + 0x3ff0020000000000ull, 0x3ff0020000000001ull, 0x3ff001ffffffffffull, 0xbff0060000000000ull, + }; + for (int e = -27; e <= 17; ++e) { + for (int k = 0; k < 8; ++k) { + uint64_t man = ((uint64_t(rng()) << 32) | rng()) & 0xfffffffffffffull; + if (k & 1) man = (man & ~0x3ffffffffffull) | 0x20000000000ull; + if (k & 2) man &= ~0x3ffffffffffull; + v.push_back((uint64_t(k & 4 ? 1 : 0) << 63) | (uint64_t(e + 1023) << 52) | man); + } + } + for (uint32_t i = 0; i < num_random; ++i) { + uint64_t e = 1023 - 30 + (rng() % 50); + uint64_t man = ((uint64_t(rng()) << 32) | rng()) & 0x800fffffffffffffull; + v.push_back(man | (e << 52)); + } + return v; +} + +void test_narrow_s(const std::vector& in) { + uint32_t n = in.size(); + Buf src(n * 4), dst(n * ZF_NUM_RM * 2), aux(n * 8); + src.write(in.data()); + std::vector out(n * ZF_NUM_RM); + std::vector mvx(n); + run(ZF_MODE_NARROW_S, n, src, dst, &aux, out.data(), dst.size); + read_aux(aux, mvx.data(), aux.size); + for (uint32_t i = 0; i < n && i < verbose; ++i) { + std::printf("in=0x%08x", in[i]); + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) + std::printf(" %04x/%04x", out[i * ZF_NUM_RM + k], rv_ftoh_s(in[i], rm_of(k), nullptr)); + std::printf("\n"); + } + for (uint32_t i = 0; i < n; ++i) { + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { + uint16_t ref = rv_ftoh_s(in[i], rm_of(k), nullptr); + if (out[i * ZF_NUM_RM + k] != ref) + report("fcvt.h.s", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } + int64_t ref = sext16(rv_ftoh_s(in[i], 0, nullptr)); + if (mvx[i] != ref) + report("fmv.x.h", in[i], 0, ref, mvx[i]); + } + std::printf("fcvt.h.s: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); +} + +void test_widen_s() { + uint32_t n = 65536; + std::vector in(n); + for (uint32_t i = 0; i < n; ++i) in[i] = i; + Buf src(n * 2), dst(n * 4); + src.write(in.data()); + std::vector out(n); + run(ZF_MODE_WIDEN_S, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + uint32_t ref = rv_htof_s(in[i], 0, nullptr); + if (out[i] != ref) + report("fcvt.s.h", in[i], 0, ref, out[i]); + } + std::printf("fcvt.s.h: all %u half inputs\n", n); +} + +void test_narrow_d(const std::vector& in) { + uint32_t n = in.size(); + Buf src(n * 8), dst(n * ZF_NUM_RM * 2); + src.write(in.data()); + std::vector out(n * ZF_NUM_RM); + run(ZF_MODE_NARROW_D, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { + uint16_t ref = rv_dtoh(in[i], rm_of(k), nullptr); + if (out[i * ZF_NUM_RM + k] != ref) + report("fcvt.h.d", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } + } + std::printf("fcvt.h.d: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); +} + +void test_widen_d() { + uint32_t n = 65536; + std::vector in(n); + for (uint32_t i = 0; i < n; ++i) in[i] = i; + Buf src(n * 2), dst(n * 8); + src.write(in.data()); + std::vector out(n); + run(ZF_MODE_WIDEN_D, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + uint64_t ref = rv_htod(in[i], 0, nullptr); + if (out[i] != ref) + report("fcvt.d.h", in[i], 0, ref, out[i]); + } + std::printf("fcvt.d.h: all %u half inputs\n", n); +} + +void test_flags(const std::vector& f32, const std::vector& f64) { + std::vector cases; + for (size_t i = 0; i < num_flag_in && i < f32.size(); ++i) + for (uint32_t rm = 0; rm < 5; ++rm) + cases.push_back({ZF_OP_H_S, rm, f32[i]}); + const uint16_t halves[] = {0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0x7bff, 0x7c00, + 0xfc00, 0x7e00, 0x7c01, 0x7d00, 0xfe00, 0xfd55, 0x7fff, 0x1234}; + for (uint16_t h : halves) + cases.push_back({ZF_OP_S_H, 0, h}); + if (xlen == 64) { + for (size_t i = 0; i < 64 && i < f64.size(); ++i) + for (uint32_t rm = 0; rm < 5; ++rm) + cases.push_back({ZF_OP_H_D, rm, f64[i]}); + for (uint16_t h : halves) + cases.push_back({ZF_OP_D_H, 0, h}); + } + uint32_t n = cases.size(); + Buf src(n * sizeof(zf_flag_case_t)), dst(n * sizeof(zf_flag_out_t)); + src.write(cases.data()); + std::vector out(n); + run(ZF_MODE_FLAGS, n, src, dst, nullptr, out.data(), dst.size); + static const char* names[] = {"fcvt.h.s", "fcvt.s.h", "fcvt.h.d", "fcvt.d.h"}; + for (uint32_t i = 0; i < n; ++i) { + const auto& c = cases[i]; + uint32_t fl = 0; + uint64_t ref = 0; + switch (c.op) { + case ZF_OP_H_S: ref = rv_ftoh_s(uint32_t(c.bits), c.rm, &fl); break; + case ZF_OP_S_H: ref = rv_htof_s(uint16_t(c.bits), c.rm, &fl); break; + case ZF_OP_H_D: ref = rv_dtoh(c.bits, c.rm, &fl); break; + case ZF_OP_D_H: ref = rv_htod(uint16_t(c.bits), c.rm, &fl); break; + } + if (out[i].result != ref) + report(names[c.op], c.bits, c.rm, ref, out[i].result); + if (out[i].fflags != fl) { + char what[32]; + std::snprintf(what, sizeof(what), "%s fflags", names[c.op]); + report(what, c.bits, c.rm, fl, out[i].fflags); + } + } + std::printf("fflags: %u single-lane cases\n", n); +} + +void test_box(std::mt19937& rng) { + uint32_t n = 1024; + std::vector h(n); + std::vector g(n); + for (uint32_t i = 0; i < n; ++i) { + h[i] = rng(); + g[i] = rng(); + if (i % 4 == 0) g[i] |= 0xffff0000; + if (i % 8 == 1) g[i] &= 0x0000ffff; + } + Buf src(n * 2), aux(n * 4), dst(n * ZF_BOX_OUTS * 8); + src.write(h.data()); + aux.write(g.data()); + std::vector out(n * ZF_BOX_OUTS); + run(ZF_MODE_BOX, n, src, dst, &aux, out.data(), dst.size); + uint64_t box = (xlen == 64) ? 0xffffffffffff0000ull : 0xffff0000ull; + for (uint32_t i = 0; i < n; ++i) { + const uint64_t* o = &out[i * ZF_BOX_OUTS]; + uint64_t r0 = box | h[i]; + uint64_t r1 = box | (g[i] & 0xffff); + uint64_t r2 = (uint64_t)sext16(h[i]); + uint64_t r3 = ((g[i] >> 16) == 0xffff) ? rv_htof_s(uint16_t(g[i]), 0, nullptr) : 0x7fc00000u; + uint64_t r4 = (uint64_t)sext16(g[i]); + if (o[0] != r0) report("flh box", h[i], 0, r0, o[0]); + if (o[1] != r1) report("fmv.h.x box", g[i], 0, r1, o[1]); + if (o[2] != r2) report("flh+fmv.x.h", h[i], 0, r2, o[2]); + if (o[3] != r3) report("unboxed fcvt.s.h", g[i], 0, r3, o[3]); + if (o[4] != r4) report("fmv.x.h unboxed", g[i], 0, r4, o[4]); + } + std::printf("boxing: %u cases\n", n); +} + +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + std::mt19937 rng(42); + + CHECK(vx_device_open(0, &dev)); + uint64_t isa = 0; + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + if (!(isa & VX_ISA_EXT_ZFHMIN)) { + std::cout << "Error: device does not support Zfhmin (VX_CFG_EXT_ZFHMIN_ENABLE)" << std::endl; + vx_device_release(dev); + return 1; + } + xlen = VX_ISA_ARCH(isa); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + CHECK(vx_queue_create(dev, &qi, &q)); + vx_module_h mod = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + std::printf("zfhmin: xlen=%u dynamic frm=%u\n", xlen, frm_dyn); + auto f32 = f32_inputs(rng); + auto f64 = f64_inputs(rng); + if (phases & 0x01) test_narrow_s(f32); + if (phases & 0x02) test_widen_s(); + if (xlen == 64) { + if (phases & 0x04) test_narrow_d(f64); + if (phases & 0x08) test_widen_d(); + } + if (phases & 0x10) test_flags(f32, f64); + if (phases & 0x20) test_box(rng); + + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} From 344039fff6ab1a72d22a6ec19113c670d39bc725 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 06:51:39 -0700 Subject: [PATCH 4/9] ci: regenerate perf baselines The MUFU and Zfhmin config keys change the resolved config hash, which marked every perf_gate baseline stale. Cycles are unchanged within 1.2% except sgemm-nt4 (+3.0%) and sgemv (nt4 +34.6%, nt16 +5.2%); sgemv already measures the new values on master (316432 instrs / 1175175 cycles at nt4), so its old baseline predates a master change. --- ci/baselines/perf/core.json | 48 +++++++++++++++---------------- ci/baselines/perf/dxa.json | 22 +++++++------- ci/baselines/perf/graphics.json | 22 +++++++------- ci/baselines/perf/raytracing.json | 6 ++-- ci/baselines/perf/tensor.json | 10 +++---- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 ++-- 8 files changed, 59 insertions(+), 59 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 77e8c4785a..dc3f8c13f1 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -6,7 +6,7 @@ }, "app": "raycast", "args": "", - "config_hash": "396cee9df6025cdb", + "config_hash": "6f10773b74cb88a7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,51 +17,51 @@ }, "app": "raycast", "args": "", - "config_hash": "262f24032ef14b58", + "config_hash": "21aa9ee5dda2e0e7", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 787850, + "cycles": 779900, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "ad2b37492d761e8f", + "config_hash": "ccecdc6a8c1dad16", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2239426, + "cycles": 2213412, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "606d011dc6f612d3", + "config_hash": "045868d9511c94f6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1838654, + "cycles": 1839049, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "9390754d3719fca3", + "config_hash": "93979463ce522b02", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5939731, + "cycles": 6115870, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "740fd84fa2938103", + "config_hash": "0f0aff075ef0f23a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "2e1c44e05f9ac34d", + "config_hash": "6ac49e61c113fe9b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,29 +83,29 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "024adb49ed96bdd3", + "config_hash": "6ffdaf4848258f2e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 685107, - "instrs": 66800 + "cycles": 720632, + "instrs": 79120 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a3d9df9695acbb95", + "config_hash": "a5e03f354713bcac", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 872926, - "instrs": 267152 + "cycles": 1175175, + "instrs": 316432 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "0e8990b25e64ad7f", + "config_hash": "28cb832d7f58703d", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "9e0c1c9767c760ea", + "config_hash": "2d4d6acd13d3f1db", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,7 +127,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "eb5fed4bfce19d76", + "config_hash": "c7f291b29cbf8558", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -138,7 +138,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "6925a5138be930ea", + "config_hash": "e2766bd823ea94d6", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,7 +149,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "b83a1e3735f79d39", + "config_hash": "c582f951f9549471", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -160,7 +160,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ef0c2548e7c5bf86", + "config_hash": "7a06e40f60aa127b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ce1eb79984a98552", + "config_hash": "b7bbdff761d2138e", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index bf1e1a010c..13e92cbb93 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,7 +6,7 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "6a815f855f6e0d05", + "config_hash": "5703e6dc69ab4754", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "626c5f6244584b1f", + "config_hash": "2631b531758393c7", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "7c32a11d123bfb13", + "config_hash": "267aa5bcabafd7e3", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "822d0ca194266a27", + "config_hash": "5bbda9802fc68b47", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "87ea544e77719254", + "config_hash": "e22911af5275076c", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "e55b61598f5f10ea", + "config_hash": "6f62080cc7b291e0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "21da2f2737bf03dd", + "config_hash": "27e92ed969c685b1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "20304652a1bfb94c", + "config_hash": "720955beafc6d6e5", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "bf4520d008c07dc9", + "config_hash": "f6dd515d7959a33c", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "2a4e017d98740636", + "config_hash": "559fa541f955de36", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f05069d7cf0ee3ff", + "config_hash": "12e6c35a774726fe", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b5ee86bfc2..b57cd0ac25 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -6,7 +6,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1e2937b20579ad3f", + "config_hash": "ae17c373f04d772f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "edc924bc4f246971", + "config_hash": "b7b7da40a1fe2413", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,29 +28,29 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "d682cc8c6f99c06b", + "config_hash": "35e77de7636aff60", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189231, + "cycles": 190255, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "46a66220c1d9956b", + "config_hash": "36f9a50d487bc61e", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740701, + "cycles": 739657, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "a7554f6fa88474dc", + "config_hash": "b7c29b8e22770b83", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2e1879d7d6f9aaf", + "config_hash": "cb1253537bc2e37f", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6e48f6b249533cc5", + "config_hash": "fe4aaf1958143ebc", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "c8c6b30537e93e2f", + "config_hash": "a948255303928571", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "a4fc4d97e6e80586", + "config_hash": "e391caeca706c831", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 1fb0ab7f7b..23aa899795 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,7 +6,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "9a4f28d2555289b5", + "config_hash": "5d32131773c11b35", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "aef578263250d585", + "config_hash": "644583aa81c236f9", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "c7b45fe5c1828bc5", + "config_hash": "8182299b41e9941e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3552c4620b..57a0a116f3 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "da0a50e46103b7dc", + "config_hash": "82de61cdb7b7428b", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2d12f1a0058fcc3b", + "config_hash": "2431723cb72c7515", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "fce886092e60e0eb", + "config_hash": "4041be1e8abcd0bb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "f0558a3b3dfde51f", + "config_hash": "b7a432cabad72ca1", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "24fd7a811fabb8c4", + "config_hash": "8edb3fc02e803b5c", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1dc1b78f41..14bf2b3b8e 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "3cfc301023342a7d", + "config_hash": "8c7905cfea98a255", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 2c4f92004c..0419ef7864 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "90defe3d20b6f739", + "config_hash": "1a56b4db8ec0d229", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index c7f771c72f..d4697ee08d 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "ba81956fe68e94b6", + "config_hash": "c21691d9c6f997b3", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "6ee14f25ec573117", + "config_hash": "dc76bfe8d0c8d0ed", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "a9ea3d8bd83a7eed", + "config_hash": "637f163388aff5e9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } From cd135b1d1b6cd9c937dec6162e26076f1a0298e2 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 07:02:16 -0700 Subject: [PATCH 5/9] fcvt: mark the overflow flag unused when no F2F format is enabled --- hw/rtl/fpu/VX_fcvt_unit.sv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/rtl/fpu/VX_fcvt_unit.sv b/hw/rtl/fpu/VX_fcvt_unit.sv index 22ef55e3e6..e0ee4ed553 100644 --- a/hw/rtl/fpu/VX_fcvt_unit.sv +++ b/hw/rtl/fpu/VX_fcvt_unit.sv @@ -564,7 +564,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; assign f2f_uf_out_s5 = f2f_uf_s5 & ~not_tiny; assign f2f_nx_s5 = is_fin_src & ((|round_sticky_bits_s5) | f2f_of_out_s5); end else begin : g_fp_pack_s - `UNUSED_VAR ({dst_h_s5, f2f_tiny0_s5, fine_round_s5}) + `UNUSED_VAR ({dst_h_s5, f2f_of_s5, f2f_tiny0_s5, fine_round_s5}) assign fp_dst_res = f32_boxed; assign f2f_nv_s5 = 1'b0; assign f2f_nx_s5 = 1'b0; From 9dbb69c13697b6851a126c2725d7bc2f17a94bda Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 16 Oct 2026 12:38:16 -0700 Subject: [PATCH 6/9] ci: regenerate perf baselines with the pinned toolchain The previous refresh was measured with a local toolchain that compiles kernels differently from vortex-toolchain-prebuilt v3.0.1. Measured with the pinned toolchain, every cycle and instruction count equals master's; only the config hashes change. --- ci/baselines/perf/core.json | 16 ++++++++-------- ci/baselines/perf/graphics.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index dc3f8c13f1..7ec4b263e4 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -23,7 +23,7 @@ }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 779900, + "cycles": 787850, "instrs": 623680 }, "app": "sgemm", @@ -34,7 +34,7 @@ }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2213412, + "cycles": 2239426, "instrs": 2494528 }, "app": "sgemm", @@ -45,7 +45,7 @@ }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1839049, + "cycles": 1838654, "instrs": 623632 }, "app": "sgemm", @@ -56,7 +56,7 @@ }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6115870, + "cycles": 5939731, "instrs": 2494480 }, "app": "sgemm", @@ -89,8 +89,8 @@ }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 720632, - "instrs": 79120 + "cycles": 685107, + "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", @@ -100,8 +100,8 @@ }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 1175175, - "instrs": 316432 + "cycles": 872926, + "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b57cd0ac25..02b67a2fe5 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -34,7 +34,7 @@ }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 190255, + "cycles": 189231, "instrs": 68624 }, "app": "gfx_om", @@ -45,7 +45,7 @@ }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 739657, + "cycles": 740701, "instrs": 274448 }, "app": "gfx_om", From 4e1162ffd8ee8f8e95f5a9db71526a96dff00a0f Mon Sep 17 00:00:00 2001 From: jel221 Date: Thu, 22 Oct 2026 15:10:47 -0700 Subject: [PATCH 7/9] mufu: move into the SFU; address review - MUFU is now an SFU processing element (SFU op 0xF): the SFU's lanes are serialized onto VX_CFG_NUM_MUFU_LANES datapath lanes (default 1). The dedicated execution unit, its dispatch/commit paths and MPM class are removed. SimX models it as a helper owned by SfuUnit. - VX_MUFU_LATENCY is a fixed constant in VX_types.toml, not a config knob. - Table ROM via a new hw/rtl/libs/VX_rom (contents as a parameter), since VX_sp_ram's init file is compiled out on the ASIC paths. - Fixes: SimX CSR writes per SFU packet; F32 NaN-box check on fp32 sources of the STD/DSP converter with FLEN=64; vx_sqrtf_fast returns a signed zero for subnormal input and +inf for +inf; F-only decode rejects FCVT forms involving D; SimX LH-to-float raw bits only under Zfhmin; MUFU test checks every op for any -r; SimX trace prints MUFU.; NaN-boxing helpers shared in sw/common/nan_box.h, consistent on RV32. - Zfhmin requires F (build-time error). - CI: 16-thread parity case; mufu DUT added to the Yosys catalog with an asic_gate baseline; fpga_gate mufu baseline re-recorded for the new DUT (fpga_gate.py --update-baseline, Vivado 2024.2); perf baselines re-recorded for the new config hash with the pinned toolchain (cycle counts unchanged). - Style: braces, 2-space indent, one parameter per line, begin/end, library priority encoder for the first active CSR lane, no cross-layer comments. --- VX_config.toml | 5 +- VX_types.toml | 11 +- ci/baselines/perf/core.json | 32 +- ci/baselines/perf/dxa.json | 22 +- ci/baselines/perf/graphics.json | 18 +- ci/baselines/perf/raytracing.json | 6 +- ci/baselines/perf/tensor.json | 10 +- ci/baselines/perf/tensor_mx.json | 2 +- ci/baselines/perf/tensor_sp.json | 2 +- ci/baselines/perf/tensor_wg.json | 6 +- ci/baselines/synthesis/xilinx/mufu.json | 205 ++-- ci/baselines/synthesis/yosys/mufu.json | 28 + ci/testcases/asic_gate.yaml | 10 + ci/testcases/fpga_gate.yaml | 4 +- ci/testcases/mufu.yaml | 19 +- docs/designs/floating_point_unit.md | 9 +- docs/hardware_library.md | 70 +- hw/dpi/dpi_float.cpp | 94 +- hw/rtl/VX_define.vh | 6 + hw/rtl/VX_gpu_pkg.sv | 30 +- hw/rtl/VX_platform.vh | 4 + hw/rtl/VX_trace_pkg.sv | 36 +- hw/rtl/core/VX_csr_data.sv | 9 - hw/rtl/core/VX_csr_unit.sv | 23 +- hw/rtl/core/VX_decode.sv | 33 +- hw/rtl/core/VX_execute.sv | 11 - hw/rtl/core/VX_sfu_unit.sv | 22 +- hw/rtl/fpu/VX_fcvt_unit.sv | 45 +- hw/rtl/libs/VX_rom.sv | 58 ++ hw/rtl/mufu/VX_mufu_pe.sv | 6 +- hw/rtl/mufu/VX_mufu_rom.sv | 1076 +++++++++++---------- hw/rtl/mufu/VX_mufu_unit.sv | 182 ++-- hw/scripts/gen_mufu_tables.py | 55 +- hw/syn/xilinx/dut/catalog.mk | 2 - hw/syn/yosys/dut/catalog.mk | 6 +- hw/unittest/fcvt_unit/main.cpp | 20 +- hw/unittest/mufu_unit/VX_mufu_unit_top.sv | 50 +- hw/unittest/mufu_unit/main.cpp | 137 +-- sim/simx/core.cpp | 23 +- sim/simx/core.h | 6 - sim/simx/csr_unit.cpp | 75 +- sim/simx/decode.cpp | 46 +- sim/simx/fpu_unit.cpp | 33 +- sim/simx/lsu_unit.cpp | 13 +- sim/simx/mufu_unit.cpp | 79 +- sim/simx/mufu_unit.h | 28 +- sim/simx/sfu_unit.cpp | 15 + sim/simx/sfu_unit.h | 6 + sim/simx/types.h | 13 - sw/common/mufu.cpp | 98 +- sw/common/mufu.h | 2 +- sw/common/nan_box.h | 56 ++ sw/kernel/include/vx_mufu.h | 46 +- sw/runtime/common/perf.cpp | 18 - tests/regression/mufu/kernel.cpp | 67 +- tests/regression/mufu/main.cpp | 90 +- tests/regression/zfhmin/kernel.cpp | 8 +- tests/regression/zfhmin/main.cpp | 113 ++- 58 files changed, 1730 insertions(+), 1469 deletions(-) create mode 100644 ci/baselines/synthesis/yosys/mufu.json create mode 100644 hw/rtl/libs/VX_rom.sv create mode 100644 sw/common/nan_box.h diff --git a/VX_config.toml b/VX_config.toml index 2e3004e6f4..ca184b4886 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -154,11 +154,8 @@ VX_CFG_FCVT_PE_RATIO = 1 VX_CFG_FNCP_PE_RATIO = 1 [mufu] -# Special-function unit (rcp/rsqrt/ex2/lg2/sinpi/cospi/tanh): one lane shared by -# all issue slots; a wider unit did not speed up fused FlashAttention. +# MUFU datapath lanes inside the SFU; the SFU's lanes are serialized onto them. VX_CFG_NUM_MUFU_LANES = 1 -VX_CFG_NUM_MUFU_BLOCKS = 1 -VX_CFG_MUFU_LATENCY = 7 [amo] # LR/SC reservation stations per LLC bank; line-indexed in BRAM, eviction yields legal spurious SC failures diff --git a/VX_types.toml b/VX_types.toml index 9ab6e5f2c6..ff3437215e 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -56,6 +56,9 @@ VX_MEM_IO_END_ADDR = "expr: $VX_MEM_USER_BASE_ADDR" VX_MEM_OM_BASE_ADDR = "expr: 0x00000000E0000000 if ($XLEN == 64) else 0xE0000000" VX_MEM_OM_END_ADDR = "expr: $VX_MEM_PAGE_TABLE_BASE_ADDR" +[mufu] +VX_MUFU_LATENCY = 7 + # VM page-table format — RISC-V architectural constants fixed by the SATP mode # (SV32 vs SV39, selected by XLEN). Relocated from VX_config.toml. [vm] @@ -592,7 +595,6 @@ VX_DCR_MPM_CLASS_TEX = 13 VX_DCR_MPM_CLASS_OM = 14 VX_DCR_MPM_CLASS_RTU = 15 VX_DCR_MPM_CLASS_DXA = 16 -VX_DCR_MPM_CLASS_MUFU = 17 [csr_mpm_base] VX_CSR_MCYCLE = 0xB00 @@ -786,13 +788,6 @@ VX_CSR_MPM_TCU_TBUF_CACHE_HITS_H = 0xB84 VX_CSR_MPM_TCU_LMEM_READS = 0xB05 # tile buffer local memory reads VX_CSR_MPM_TCU_LMEM_READS_H = 0xB85 -[csr_mpm_mufu] -# PERF: special-function unit (per-core; summed across issue slots) -VX_CSR_MPM_MUFU_STALLS = 0xB03 # dispatch stalled due to MUFU pipeline full -VX_CSR_MPM_MUFU_STALLS_H = 0xB83 -VX_CSR_MPM_MUFU_INSTRS = 0xB04 # total MUFU instructions -VX_CSR_MPM_MUFU_INSTRS_H = 0xB84 - [csr_mpm_tex] # PERF: texture unit (per-core; summed across blocks) VX_CSR_MPM_TEX_READS = 0xB03 # texture accesses diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 7ec4b263e4..8467f80317 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -6,7 +6,7 @@ }, "app": "raycast", "args": "", - "config_hash": "6f10773b74cb88a7", + "config_hash": "30868719eb39dd56", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "raycast", "args": "", - "config_hash": "21aa9ee5dda2e0e7", + "config_hash": "bbb5e759fb64a2d2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "ccecdc6a8c1dad16", + "config_hash": "f3cdbb1a54b60bd7", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "045868d9511c94f6", + "config_hash": "1f13c513ce419a18", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "93979463ce522b02", + "config_hash": "6c4c16bd5ba783fb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "0f0aff075ef0f23a", + "config_hash": "e654782241f3196f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "6ac49e61c113fe9b", + "config_hash": "406f5c17ca35ee9d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "6ffdaf4848258f2e", + "config_hash": "5016dd4b1102f6fc", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a5e03f354713bcac", + "config_hash": "b631a0bafc91d264", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "28cb832d7f58703d", + "config_hash": "14333df532bad1d6", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "2d4d6acd13d3f1db", + "config_hash": "fdbcac76ee256139", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,7 +127,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "c7f291b29cbf8558", + "config_hash": "d1489b9fccb46500", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -138,7 +138,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "e2766bd823ea94d6", + "config_hash": "7514f2e770fda7af", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,7 +149,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "c582f951f9549471", + "config_hash": "7b1283f03793abb6", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -160,7 +160,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "7a06e40f60aa127b", + "config_hash": "3c79fc19ad8549a0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "b7bbdff761d2138e", + "config_hash": "75b65abb22c3753b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index 13e92cbb93..a57e2798f3 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,7 +6,7 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "5703e6dc69ab4754", + "config_hash": "1d98c347f39baa17", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "2631b531758393c7", + "config_hash": "f0ace578558d01a1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "267aa5bcabafd7e3", + "config_hash": "9a3f1b02710f7458", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "5bbda9802fc68b47", + "config_hash": "4d3b7f000ce31563", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "e22911af5275076c", + "config_hash": "4dffe3629adfe705", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "6f62080cc7b291e0", + "config_hash": "6aac48586a19e5d1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "27e92ed969c685b1", + "config_hash": "15066287b6cf674a", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "720955beafc6d6e5", + "config_hash": "6f3f943387268cd9", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "f6dd515d7959a33c", + "config_hash": "99f4567a3f3d04a6", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "559fa541f955de36", + "config_hash": "a45156b86aef0add", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "12e6c35a774726fe", + "config_hash": "63afe161fb9d2aac", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 02b67a2fe5..761b756cb9 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -6,7 +6,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ae17c373f04d772f", + "config_hash": "71e067fd7765f075", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "b7b7da40a1fe2413", + "config_hash": "ac94e8341d7c171e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "35e77de7636aff60", + "config_hash": "00c3aa3428bf4e86", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "36f9a50d487bc61e", + "config_hash": "a99eb5442dfe8bd5", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "b7c29b8e22770b83", + "config_hash": "b1631e4a43822c5a", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "cb1253537bc2e37f", + "config_hash": "2c54468d2adf668e", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "fe4aaf1958143ebc", + "config_hash": "1f50896f410bc97f", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "a948255303928571", + "config_hash": "b72e550c7f6c7aca", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "e391caeca706c831", + "config_hash": "15f81408ca91fc9f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 23aa899795..db73f213c5 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,7 +6,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "5d32131773c11b35", + "config_hash": "048a348462ce8158", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "644583aa81c236f9", + "config_hash": "bbab935bba11ff85", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "8182299b41e9941e", + "config_hash": "f2e45a7a7018b151", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 57a0a116f3..85797034f5 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "82de61cdb7b7428b", + "config_hash": "d538172b26e2967c", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2431723cb72c7515", + "config_hash": "02375c8f3588b56d", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "4041be1e8abcd0bb", + "config_hash": "38834e790f4f6a3f", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b7a432cabad72ca1", + "config_hash": "fcf4a952a5f93107", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "8edb3fc02e803b5c", + "config_hash": "66b2e78e9663c54b", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 14bf2b3b8e..a9ed6b1e98 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "8c7905cfea98a255", + "config_hash": "eac1b740108b63ac", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 0419ef7864..d898fae929 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "1a56b4db8ec0d229", + "config_hash": "1ab2d7ff293df75a", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index d4697ee08d..1caa93120b 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "c21691d9c6f997b3", + "config_hash": "23715dbc37d808e1", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "dc76bfe8d0c8d0ed", + "config_hash": "3eafd3c24fc1aba9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "637f163388aff5e9", + "config_hash": "448955a5824dd803", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/synthesis/xilinx/mufu.json b/ci/baselines/synthesis/xilinx/mufu.json index 1faf73164c..2e4c0076a4 100644 --- a/ci/baselines/synthesis/xilinx/mufu.json +++ b/ci/baselines/synthesis/xilinx/mufu.json @@ -6,107 +6,107 @@ "dut": "mufu", "result": { "bram": 1, - "build_time_s": 885, + "build_time_s": 796, "critical_paths": [ { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__33/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", "group": "core_clock", - "levels": 8, - "logic_ns": 0.924, - "route_ns": 1.721, - "route_pct": 65.069, - "slack_ns": 0.668, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.126, + "route_ns": 1.709, + "route_pct": 60.288, + "slack_ns": 0.381, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__33/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", "group": "core_clock", - "levels": 11, - "logic_ns": 0.984, - "route_ns": 1.652, - "route_pct": 62.675, - "slack_ns": 0.677, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.127, + "route_ns": 1.707, + "route_pct": 60.239, + "slack_ns": 0.382, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__34/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", "group": "core_clock", - "levels": 8, - "logic_ns": 0.9, - "route_ns": 1.723, - "route_pct": 65.691, - "slack_ns": 0.69, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.128, + "route_ns": 1.705, + "route_pct": 60.18, + "slack_ns": 0.383, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__32/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", "group": "core_clock", - "levels": 8, - "logic_ns": 0.894, - "route_ns": 1.721, - "route_pct": 65.815, - "slack_ns": 0.698, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.126, + "route_ns": 1.706, + "route_pct": 60.246, + "slack_ns": 0.384, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__34/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", "group": "core_clock", - "levels": 11, - "logic_ns": 0.96, - "route_ns": 1.654, - "route_pct": 63.279, - "slack_ns": 0.699, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.126, + "route_ns": 1.706, + "route_pct": 60.246, + "slack_ns": 0.384, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__31/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", "group": "core_clock", - "levels": 8, - "logic_ns": 0.89, - "route_ns": 1.722, - "route_pct": 65.929, - "slack_ns": 0.701, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.127, + "route_ns": 1.704, + "route_pct": 60.197, + "slack_ns": 0.385, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__32/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", "group": "core_clock", - "levels": 11, - "logic_ns": 0.954, - "route_ns": 1.652, - "route_pct": 63.396, - "slack_ns": 0.707, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.127, + "route_ns": 1.704, + "route_pct": 60.197, + "slack_ns": 0.385, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__29/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", "group": "core_clock", - "levels": 8, - "logic_ns": 0.884, - "route_ns": 1.721, - "route_pct": 66.068, - "slack_ns": 0.708, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.128, + "route_ns": 1.702, + "route_pct": 60.138, + "slack_ns": 0.386, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__42/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", "group": "core_clock", - "levels": 7, - "logic_ns": 0.909, - "route_ns": 1.696, - "route_pct": 65.108, - "slack_ns": 0.708, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.128, + "route_ns": 1.702, + "route_pct": 60.138, + "slack_ns": 0.386, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" }, { - "endpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r_reg[94]_fret__2_fret__40/D", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", "group": "core_clock", - "levels": 7, - "logic_ns": 0.909, - "route_ns": 1.695, - "route_pct": 65.095, - "slack_ns": 0.709, - "startpoint": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg/C" + "levels": 12, + "logic_ns": 1.125, + "route_ns": 1.703, + "route_pct": 60.216, + "slack_ns": 0.388, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" } ], "dsp": 3, @@ -116,59 +116,64 @@ "vivado": "2024.2", "xlen": 32 }, - "ff": 1153, - "fmax_mhz": 375.2, + "ff": 2278, + "fmax_mhz": 338.8, "high_fanout_nets": [ { - "driver": "LUT3", - "fanout": 608, - "net": "mufu_unit/g_blocks[0].pe_serializer/out_buf/g_eb1.pipe_buffer/g_register.g_pipe_regs[0].pipe_register/g_pipe.g_partial_reset.pipe_rst_reg[0][0]_1" + "driver": "FDRE", + "fanout": 1556, + "net": "mufu_unit/pe_serializer/shift_reg/pe_valid_in" }, { "driver": "LUT2", - "fanout": 233, - "net": "mufu_unit/g_blocks[0].pe_serializer/out_buf/g_eb1.pipe_buffer/g_register.g_pipe_regs[0].pipe_register/g_buffer.flow_out" + "fanout": 718, + "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep_0" }, { - "driver": "LUT5", - "fanout": 149, - "net": "mufu_unit/lane_dispatch/g_blocks[0].g_partial_simd.packet_iter/g_iterator.pipe_reg/g_iterator.sent_mask_p_reg[7][1]" + "driver": "LUT2", + "fanout": 583, + "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.fire_in" }, { - "driver": "LUT5", - "fanout": 149, - "net": "mufu_unit/lane_dispatch/g_blocks[0].g_partial_simd.packet_iter/g_iterator.pipe_reg/g_iterator.sent_mask_p_reg[7][0]" + "driver": "LUT2", + "fanout": 583, + "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.flow_out" }, { - "driver": "FDRE", - "fanout": 111, - "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/per_block_execute_if[0]\\\\.data[op_type][1]" + "driver": "MUXF8", + "fanout": 164, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[24]" }, { - "driver": "LUT3", - "fanout": 108, - "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r[88]_i_1_n_0" + "driver": "MUXF8", + "fanout": 162, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[23]" }, { "driver": "FDRE", - "fanout": 107, - "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/per_block_execute_if[0]\\\\.data[op_type][0]" + "fanout": 162, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx[3]" + }, + { + "driver": "MUXF8", + "fanout": 148, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[25]" }, { "driver": "FDSE", - "fanout": 106, - "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_0" + "fanout": 146, + "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep_n_0" }, { - "driver": "LUT3", - "fanout": 104, - "net": "mufu_unit/lane_dispatch/g_blocks[0].buf_out/g_eb2.stream_buffer/g_buffer.g_out_reg.data_out_r[87]_i_1_n_0" + "driver": "FDSE", + "fanout": 143, + "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep__3_n_0" } ], - "lut": 1227, - "lutram": 57, + "lut": 2624, + "lutram": 64, "uram": 0, - "wns_ns": 0.668 + "wns_ns": 0.381 } } } diff --git a/ci/baselines/synthesis/yosys/mufu.json b/ci/baselines/synthesis/yosys/mufu.json new file mode 100644 index 0000000000..e9740effe8 --- /dev/null +++ b/ci/baselines/synthesis/yosys/mufu.json @@ -0,0 +1,28 @@ +{ + "mufu": { + "clock_mhz": 800, + "config_hash": "ec81347c2979f17c", + "configs": "", + "dut": "mufu", + "result": { + "build_time_s": 130, + "cell_area_um2": 2700.974, + "cell_count": 26212, + "env": { + "corner": "tt", + "pdk": "asap7", + "sta": "3.1.0", + "sv2v": "sv2v v0.0.13-13-g493a88f", + "vt": "rvt", + "xlen": 32, + "yosys": "Yosys 0.69" + }, + "fmax_mhz": 1069.519, + "power_mw": 19.1, + "seq_area_um2": 847.973, + "sram_area_um2": 0.0, + "tns_ns": 0.0, + "wns_ns": 0.315 + } + } +} diff --git a/ci/testcases/asic_gate.yaml b/ci/testcases/asic_gate.yaml index 5f81318062..00def8e7cc 100644 --- a/ci/testcases/asic_gate.yaml +++ b/ci/testcases/asic_gate.yaml @@ -136,6 +136,16 @@ builds: clock_mhz: 800 configs: "" +# --- mufu -------------------------------------------------------------------- + +# Special-function datapath at its default single lane, serialized from a +# 16-lane SFU: the 7-stage table+polynomial pipeline and the 512x72 table ROM. +- id: mufu + group: mufu + dut: mufu + clock_mhz: 800 + configs: "" + # --- tensor ------------------------------------------------------------------ # TFR tensor core with every datatype plus sparsity and WGMMA. The widest diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index 0905d5503a..cae9b0f0a8 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -105,8 +105,8 @@ builds: # --- mufu -------------------------------------------------------------------- -# Special-function unit at its default single lane under a 16-wide warp: lane -# dispatch, the 7-stage table+polynomial datapath and the 512x72 table ROM. +# Special-function datapath at its default single lane, serialized from a +# 16-lane SFU: the 7-stage table+polynomial pipeline and the 512x72 table ROM. - id: mufu group: mufu dut: mufu diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml index c17a5d5589..c7f70873b9 100644 --- a/ci/testcases/mufu.yaml +++ b/ci/testcases/mufu.yaml @@ -8,14 +8,14 @@ defaults: - 32 - 64 tier: smoke - touches: [hw/rtl/mufu, sim/simx/mufu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] + touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] tests: - id: mufu-1 via: blackbox drivers: - simx app: mufu -# Wide warp: 16 threads over 1 MUFU lane, so every warp serializes into 16 packets. +# Wide warp: 16 threads serialized onto 1 MUFU lane, 16 issue cycles per warp. - id: mufu-nt16 via: blackbox drivers: @@ -23,7 +23,7 @@ tests: - rtlsim app: mufu configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -# Full-rate lanes (non-default): no lane-gather stage on the response path. +# One MUFU lane per SFU lane (non-default): nothing is serialized. - id: mufu-full-lanes via: blackbox drivers: @@ -42,18 +42,23 @@ tests: # >=~300k cycles so steady-state MUFU throughput dominates. # --------------------------------------------------------------------------- -# Default shape: 4 threads over the 1 lane, chained ex2 (~609k cycles). +# Default shape: 4 threads over the 1 lane, repeated ex2 (~620k cycles). - id: parity-mufu check: model_parity via: blackbox app: mufu args: -n1024 -r256 -tex2 -# 8 threads over 1 lane (8 packets per warp), chained sinpi (~670k cycles). -# Not 16 threads: there SimX runs ~5% fast on saturated multi-packet MUFU -# issue. +# 8 threads serialized onto the 1 lane, repeated sinpi (~547k cycles). - id: parity-mufu-nt8 check: model_parity via: blackbox app: mufu args: -n2048 -r256 -tsinpi configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 +# 16 threads serialized onto the 1 lane, the fpga_gate shape (~532k cycles). +- id: parity-mufu-nt16 + check: model_parity + via: blackbox + app: mufu + args: -n2048 -r256 -tsinpi + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index a4e2fda7fd..7e9e9d4eb0 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -457,10 +457,11 @@ and it is F32 only: the F64 soft cores are covered by the unit tests alone. ### Special functions are not in the FPU Approximate transcendentals (`rcp`, `rsqrt`, `ex2`, `lg2`, `sinpi`, `cospi`, -`tanh`) run on a separate execution unit, the MUFU, not on an FPU core. It has -its own dispatch queue and commit path, so FPU and MUFU instructions from -different warps execute concurrently; it ignores `frm` and never writes -`fflags`. The FPU's `fsqrt`/`fdiv` remain the correctly rounded operations. +`tanh`) run on the MUFU, a processing element of the SFU, not on an FPU core. +The SFU's lanes are serialized onto `VX_CFG_NUM_MUFU_LANES` datapath lanes +(default 1), so FPU and MUFU instructions from different warps execute +concurrently; it ignores `frm` and never writes `fflags`. The FPU's +`fsqrt`/`fdiv` remain the correctly rounded operations. --- diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 6436335429..0ba4a8bdb5 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -2,7 +2,7 @@ ## Overview -The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 75 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. +The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 76 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. All modules are parameterized for data width, depth, and implementation strategy, with support for both FPGA (Xilinx/Intel) and ASIC synthesis flows. @@ -1260,6 +1260,25 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn --- +### VX_rom + +**File:** `VX_rom.sv` +**Purpose:** Read-only memory whose contents are a packed parameter, so it needs no init file. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DATAW` | 1 | Data width in bits | +| `SIZE` | 1 | Number of entries | +| `OUT_REG` | 0 | Registered read (one cycle) instead of an asynchronous read | +| `INIT_DATA` | `'0` | Contents, `SIZE*DATAW` bits; entry `i` is `INIT_DATA[i*DATAW +: DATAW]` | + +**Features:** +- Contents are elaborated as constants, so they survive the ASIC flow, where `VX_sp_ram`'s `INIT_FILE` loading is compiled out +- Under synthesis, a registered ROM that passes `FORCE_BRAM` is tagged for block ROM (`rom_style`/`romstyle`); otherwise it maps to logic +- Generated tables (e.g. `hw/rtl/mufu/VX_mufu_rom.sv`) instantiate it directly + +--- + ### VX_async_ram_patch **File:** `VX_async_ram_patch.sv` @@ -1858,27 +1877,28 @@ Infrastructure for runtime debug observation and serial trace extraction. | 49 | VX_priority_encoder | Encoder | Small | O(log N) | | 50 | VX_reduce_tree | Arithmetic | Small | O(log N) | | 51 | VX_reset_relay | Sync | Small | 1 cycle | -| 52 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | -| 53 | VX_scan | Encoder | Medium | O(log N) | -| 54 | VX_scope_switch | Debug | Small | 0 cycles | -| 55 | VX_scope_tap | Debug | Large | Multi-cycle | -| 56 | VX_serial_div | Arithmetic | Small | Multi-cycle | -| 57 | VX_serial_mul | Arithmetic | Small | Multi-cycle | -| 58 | VX_shift_register | Buffer | Varies | N cycles | -| 59 | VX_skid_buffer | Buffer | Small | 0-1 cycles | -| 60 | VX_sp_ram | Memory | Varies | 0-1 cycles | -| 61 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | -| 62 | VX_stream_buffer | Buffer | Small | 0-1 cycles | -| 63 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | -| 64 | VX_stream_fork | Stream Util | Small | 0 cycles | -| 65 | VX_stream_join | Stream Util | Small | 0 cycles | -| 66 | VX_stream_omega | Interconnect | Large | O(log N) | -| 67 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | -| 68 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | -| 69 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | -| 70 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | -| 71 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | -| 72 | VX_ticket_lock | Allocator | Small | 0-1 cycles | -| 73 | VX_toggle_buffer | Buffer | Small | 1 cycle | -| 74 | VX_transpose | Bit Manip | Zero | 0 cycles | -| 75 | VX_wallace_mul | Arithmetic | Large | O(log N) | +| 52 | VX_rom | Memory | Varies | 0-1 cycles | +| 53 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | +| 54 | VX_scan | Encoder | Medium | O(log N) | +| 55 | VX_scope_switch | Debug | Small | 0 cycles | +| 56 | VX_scope_tap | Debug | Large | Multi-cycle | +| 57 | VX_serial_div | Arithmetic | Small | Multi-cycle | +| 58 | VX_serial_mul | Arithmetic | Small | Multi-cycle | +| 59 | VX_shift_register | Buffer | Varies | N cycles | +| 60 | VX_skid_buffer | Buffer | Small | 0-1 cycles | +| 61 | VX_sp_ram | Memory | Varies | 0-1 cycles | +| 62 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | +| 63 | VX_stream_buffer | Buffer | Small | 0-1 cycles | +| 64 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | +| 65 | VX_stream_fork | Stream Util | Small | 0 cycles | +| 66 | VX_stream_join | Stream Util | Small | 0 cycles | +| 67 | VX_stream_omega | Interconnect | Large | O(log N) | +| 68 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | +| 69 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | +| 70 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | +| 71 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | +| 72 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | +| 73 | VX_ticket_lock | Allocator | Small | 0-1 cycles | +| 74 | VX_toggle_buffer | Buffer | Small | 1 cycle | +| 75 | VX_transpose | Bit Manip | Zero | 0 cycles | +| 76 | VX_wallace_mul | Arithmetic | Large | O(log N) | diff --git a/hw/dpi/dpi_float.cpp b/hw/dpi/dpi_float.cpp index 20ae399c81..5e59e6559b 100644 --- a/hw/dpi/dpi_float.cpp +++ b/hw/dpi/dpi_float.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "svdpi.h" #include "verilated_vpi.h" @@ -51,117 +52,48 @@ extern "C" { void dpi_fmax(bool enable, int dst_fmt, int64_t a, int64_t b, int64_t* result, svBitVecVal* fflags); } +constexpr uint32_t kRegBits = VX_CFG_XLEN; + inline uint64_t nan_box(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffff00000000; -#else - return value; -#endif + return nanbox::box(value, 32, kRegBits); } inline uint64_t nan_box16(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffffffff0000; -#else - return (value & 0xffff) | 0xffff0000; -#endif + return nanbox::box(value, 16, kRegBits); } inline uint64_t nan_box19(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xfffffffffff80000; -#else - return value; -#endif + return nanbox::box(value, 19, kRegBits); } inline uint64_t nan_box8(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffffffffff00; -#else - return value; -#endif + return nanbox::box(value, 8, kRegBits); } inline uint64_t nan_box4(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xfffffffffffffff0; -#else - return value; -#endif -} - -inline bool is_nan_boxed(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint32_t(value >> 32) == 0xffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed16(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 16) == 0xffffffffffff); -#else - return ((uint32_t(value) >> 16) == 0xffff); -#endif -} - -inline bool is_nan_boxed19(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 19) == 0xffffffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed8(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 8) == 0xffffffffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed4(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 4) == 0xfffffffffffffff); -#else - return true; -#endif + return nanbox::box(value, 4, kRegBits); } inline int64_t check_boxing(int64_t a) { - if (is_nan_boxed(a)) - return a; - return nan_box(0x7fc00000); // NaN + return nanbox::is_boxed(a, 32, kRegBits) ? a : nan_box(0x7fc00000); } inline int64_t check_boxing16(int64_t a, uint16_t nan_value) { - if (is_nan_boxed16(a)) - return a; - return nan_box16(nan_value); // NaN + return nanbox::is_boxed(a, 16, kRegBits) ? a : nan_box16(nan_value); } inline int64_t check_boxing19(int64_t a, uint32_t nan_value) { - if (is_nan_boxed19(a)) - return a; - return nan_box19(nan_value); // NaN + return nanbox::is_boxed(a, 19, kRegBits) ? a : nan_box19(nan_value); } inline int64_t check_boxing8(int64_t a, uint8_t nan_value) { - if (is_nan_boxed8(a)) - return a; - return nan_box8(nan_value); // NaN + return nanbox::is_boxed(a, 8, kRegBits) ? a : nan_box8(nan_value); } inline int64_t check_boxing4(int64_t a, uint8_t nan_value) { - if (is_nan_boxed4(a)) - return a; - return nan_box4(nan_value); // NaN + return nanbox::is_boxed(a, 4, kRegBits) ? a : nan_box4(nan_value); } - void dpi_fadd(bool enable, int dst_fmt, int64_t a, int64_t b, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags) { if (!enable) return; diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 0dfd4dffe1..7607b2bdf6 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -101,6 +101,12 @@ `endif `endif +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + `ifndef VX_CFG_EXT_F_ENABLE + `error "VX_CFG_EXT_ZFHMIN_ENABLE requires VX_CFG_EXT_F_ENABLE" + `endif +`endif + // Convenience flag: the TCU metadata SRAM is present when any metadata-consuming // mode (MX or sparse) is enabled. Internal derived macro — not a VX_CFG_* knob. `ifdef VX_CFG_TCU_MX_ENABLE diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index b66e7ac049..53193421c1 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -240,9 +240,8 @@ package VX_gpu_pkg; localparam EX_SFU = 2; localparam EX_FPU = (EX_SFU + `VX_CFG_EXT_F_ENABLED); localparam EX_TCU = (EX_FPU + `VX_CFG_EXT_TCU_ENABLED); - localparam EX_MUFU = (EX_TCU + `VX_CFG_EXT_MUFU_ENABLED); - localparam NUM_EX_UNITS = EX_MUFU + 1; + localparam NUM_EX_UNITS = EX_TCU + 1; localparam EX_BITS = `CLOG2(NUM_EX_UNITS); localparam EX_WIDTH = `UP(EX_BITS); @@ -527,6 +526,9 @@ package VX_gpu_pkg; `endif `ifdef EXT_GFX_ANY_ENABLE localparam INST_SFU_RTUW = 4'hE; // RTU hit window (GETW/GETWF) + trace ops +`endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam INST_SFU_MUFU = 4'hF; `endif localparam INST_SFU_BITS = 4; @@ -549,15 +551,14 @@ package VX_gpu_pkg; endfunction `ifdef VX_CFG_EXT_MUFU_ENABLE - localparam INST_MUFU_RCP = 4'h0; - localparam INST_MUFU_RSQRT = 4'h1; - localparam INST_MUFU_EX2 = 4'h2; - localparam INST_MUFU_LG2 = 4'h3; - localparam INST_MUFU_SINPI = 4'h4; - localparam INST_MUFU_COSPI = 4'h5; - localparam INST_MUFU_TANH = 4'h6; - localparam INST_MUFU_BITS = 4; - localparam INST_MUFU_FMT_BITS = 3; + localparam INST_MUFU_RCP = 3'h0; + localparam INST_MUFU_RSQRT = 3'h1; + localparam INST_MUFU_EX2 = 3'h2; + localparam INST_MUFU_LG2 = 3'h3; + localparam INST_MUFU_SINPI = 3'h4; + localparam INST_MUFU_COSPI = 3'h5; + localparam INST_MUFU_TANH = 3'h6; + localparam INST_MUFU_BITS = 3; `endif /////////////////////////////// Issue parameters ////////////////////////// @@ -975,8 +976,8 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_MUFU_ENABLE typedef struct packed { - logic [INST_ARGS_BITS-INST_MUFU_FMT_BITS-1:0] __padding; - logic [INST_MUFU_FMT_BITS-1:0] fmt; + logic [INST_ARGS_BITS-INST_MUFU_BITS-1:0] __padding; + logic [INST_MUFU_BITS-1:0] fn; } mufu_args_t; `PACKAGE_ASSERT($bits(mufu_args_t) == INST_ARGS_BITS) `endif @@ -1219,9 +1220,6 @@ package VX_gpu_pkg; `DECL_EXECUTE_T (alu, `VX_CFG_NUM_ALU_LANES); `DECL_EXECUTE_T (lsu, `VX_CFG_NUM_LSU_LANES); `DECL_EXECUTE_T (sfu, `VX_CFG_NUM_SFU_LANES); -`ifdef VX_CFG_EXT_MUFU_ENABLE - `DECL_EXECUTE_T (mufu, `VX_CFG_NUM_MUFU_LANES); -`endif //////////////////////////// Perf counter types /////////////////////////// diff --git a/hw/rtl/VX_platform.vh b/hw/rtl/VX_platform.vh index 9b4ddd636d..85fae1d635 100644 --- a/hw/rtl/VX_platform.vh +++ b/hw/rtl/VX_platform.vh @@ -170,6 +170,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM (* ramstyle = "block" *) +`define USE_BLOCK_ROM (* romstyle = "M20K" *) `define USE_FAST_BRAM (* ramstyle = "MLAB, no_rw_check" *) `define NO_RW_RAM_CHECK (* altera_attribute = "-name add_pass_through_logic_to_inferred_rams off" *) `define RW_RAM_CHECK (* altera_attribute = "-name add_pass_through_logic_to_inferred_rams on" *) @@ -183,6 +184,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM (* ram_style = "block" *) +`define USE_BLOCK_ROM (* rom_style = "block" *) `define USE_FAST_BRAM (* ram_style = "distributed" *) `define NO_RW_RAM_CHECK (* rw_addr_collision = "no" *) `define RW_RAM_CHECK (* rw_addr_collision = "yes" *) @@ -199,6 +201,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) 0 `define USE_BLOCK_BRAM +`define USE_BLOCK_ROM `define USE_FAST_BRAM `define NO_RW_RAM_CHECK `define RW_RAM_CHECK @@ -212,6 +215,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM +`define USE_BLOCK_ROM `define USE_FAST_BRAM `define NO_RW_RAM_CHECK `define RW_RAM_CHECK diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index cc2d753f56..8c3f8933c1 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -36,9 +36,6 @@ package VX_trace_pkg; `endif `ifdef VX_CFG_EXT_TCU_ENABLE EX_TCU: `TRACE(level, ("TCU")) - `endif - `ifdef VX_CFG_EXT_MUFU_ENABLE - EX_MUFU: `TRACE(level, ("MUFU")) `endif default: `TRACE(level, ("?")) endcase @@ -272,6 +269,20 @@ package VX_trace_pkg; `endif `ifdef VX_CFG_EXT_RASTER_ENABLE INST_SFU_RASTER: `TRACE(level, ("RAST.FETCH")) + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + INST_SFU_MUFU: begin + case (op_args.mufu.fn) + INST_MUFU_RCP: `TRACE(level, ("MUFU.RCP")) + INST_MUFU_RSQRT: `TRACE(level, ("MUFU.RSQRT")) + INST_MUFU_EX2: `TRACE(level, ("MUFU.EX2")) + INST_MUFU_LG2: `TRACE(level, ("MUFU.LG2")) + INST_MUFU_SINPI: `TRACE(level, ("MUFU.SINPI")) + INST_MUFU_COSPI: `TRACE(level, ("MUFU.COSPI")) + INST_MUFU_TANH: `TRACE(level, ("MUFU.TANH")) + default: `TRACE(level, ("MUFU.?")) + endcase + end `endif default: `TRACE(level, ("?")) endcase @@ -442,20 +453,6 @@ package VX_trace_pkg; EX_TCU: begin VX_tcu_pkg::trace_ex_op(level, op_type, op_args); end - `endif - `ifdef VX_CFG_EXT_MUFU_ENABLE - EX_MUFU: begin - case (INST_MUFU_BITS'(op_type)) - INST_MUFU_RCP: `TRACE(level, ("MUFU.RCP")) - INST_MUFU_RSQRT: `TRACE(level, ("MUFU.RSQRT")) - INST_MUFU_EX2: `TRACE(level, ("MUFU.EX2")) - INST_MUFU_LG2: `TRACE(level, ("MUFU.LG2")) - INST_MUFU_SINPI: `TRACE(level, ("MUFU.SINPI")) - INST_MUFU_COSPI: `TRACE(level, ("MUFU.COSPI")) - INST_MUFU_TANH: `TRACE(level, ("MUFU.TANH")) - default: `TRACE(level, ("?")) - endcase - end `endif default: `TRACE(level, ("?")) endcase @@ -482,11 +479,6 @@ package VX_trace_pkg; EX_FPU: begin `TRACE(level, (", fmt=0x%0h, frm=0x%0h", op_args.fpu.fmt, op_args.fpu.frm)) end - `endif - `ifdef VX_CFG_EXT_MUFU_ENABLE - EX_MUFU: begin - `TRACE(level, (", fmt=0x%0h", op_args.mufu.fmt)) - end `endif default:; endcase diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 668c5f7ac0..f09c17bb7e 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -529,15 +529,6 @@ import VX_fpu_pkg::*; endcase end `endif - `ifdef VX_CFG_EXT_MUFU_ENABLE - `VX_DCR_MPM_CLASS_MUFU: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_MUFU_STALLS, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_MUFU]); - `CSR_READ_64(`VX_CSR_MPM_MUFU_INSTRS, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_MUFU]); - default:; - endcase - end - `endif `ifdef VX_CFG_EXT_TEX_ENABLE `VX_DCR_MPM_CLASS_TEX: begin case (dcr_addr) diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index e6e2f70334..ae0b7038b2 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -72,14 +72,21 @@ module VX_csr_unit import VX_gpu_pkg::*; #( assign rs1_data[i] = execute_if.data.rs1_data[i]; end - reg [`VX_CFG_XLEN-1:0] rs1_first; - always @(*) begin - rs1_first = rs1_data[0]; - for (integer i = NUM_LANES-1; i >= 0; --i) begin - if (execute_if.data.header.tmask[i]) begin - rs1_first = rs1_data[i]; - end - end + wire [`VX_CFG_XLEN-1:0] rs1_first; + if (NUM_LANES > 1) begin : g_rs1_first + wire [`LOG2UP(NUM_LANES)-1:0] first_lane; + VX_priority_encoder #( + .N (NUM_LANES), + .REVERSE (0) + ) first_lane_sel ( + .data_in (execute_if.data.header.tmask), + .index_out (first_lane), + `UNUSED_PIN (onehot_out), + `UNUSED_PIN (valid_out) + ); + assign rs1_first = rs1_data[first_lane]; + end else begin : g_rs1_first_0 + assign rs1_first = rs1_data[0]; end wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_first; diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index f5775817fe..12a59ca372 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -61,6 +61,15 @@ module VX_decode import wire [4:0] rs3 = instr[31:27]; `UNUSED_VAR (funct2) + +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire fcvt_fmt_valid = (rs2[4:2] == 3'b000) && (rs2[1:0] != 2'b11) && (funct2 != 2'b11) && (rs2[1:0] != funct2); +`ifdef VX_CFG_FLEN_64 + wire fcvt_fmt_supported = fcvt_fmt_valid; +`else + wire fcvt_fmt_supported = fcvt_fmt_valid && (rs2[1:0] != 2'b01) && (funct2 != 2'b01); +`endif +`endif `ifndef VX_CFG_EXT_A_ENABLE `UNUSED_VAR (funct5) `endif @@ -508,15 +517,17 @@ module VX_decode import end `ifdef VX_CFG_EXT_ZFHMIN_ENABLE 5'b01000: begin - if (funct2 == 2'b10) begin - op_type = INST_OP_BITS'(INST_FPU_F2H); - op_args.fpu.fmt[0] = rs2[0]; - end else begin - op_type = INST_OP_BITS'(INST_FPU_F2F); + if (fcvt_fmt_supported) begin + if (funct2 == 2'b10) begin + op_type = INST_OP_BITS'(INST_FPU_F2H); + op_args.fpu.fmt[0] = rs2[0]; + end else begin + op_type = INST_OP_BITS'(INST_FPU_F2F); + end + rd_xregs[XREG_1] = frm_is_dyn; + `USED_FREG (rd); + `USED_FREG (rs1); end - rd_xregs[XREG_1] = frm_is_dyn; - `USED_FREG (rd); - `USED_FREG (rs1); end `elsif VX_CFG_FLEN_64 5'b01000: begin @@ -901,10 +912,10 @@ module VX_decode import `ifdef VX_CFG_EXT_MUFU_ENABLE INST_EXT3: begin if (funct7 == 7'h00 && funct3 == 3'h0 && rs2 <= 5'(INST_MUFU_TANH)) begin - ex_type = EX_MUFU; - op_type = INST_OP_BITS'(rs2); + ex_type = EX_SFU; + op_type = INST_OP_BITS'(INST_SFU_MUFU); op_args.mufu.__padding = '0; - op_args.mufu.fmt = funct3; + op_args.mufu.fn = INST_MUFU_BITS'(rs2); `USED_FREG (rd); `USED_FREG (rs1); end diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index f7eb8fec39..4f0222e066 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -142,17 +142,6 @@ module VX_execute import VX_gpu_pkg::*; #( ); `endif -`ifdef VX_CFG_EXT_MUFU_ENABLE - VX_mufu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))) - ) mufu_unit ( - .clk (clk), - .reset (reset), - .dispatch_if (dispatch_if[EX_MUFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .commit_if (commit_if[EX_MUFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) - ); -`endif - VX_sfu_unit #( .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))), .CORE_ID (CORE_ID) diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 660303ee31..d7a200d11a 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -68,7 +68,7 @@ import VX_raster_pkg::*; `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = 1; localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; - localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; + localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_MUFU_ENABLED; localparam PE_SEL_BITS = `CLOG2(PE_COUNT); localparam PE_IDX_WCTL = 0; localparam PE_IDX_CSRS = 1; @@ -87,6 +87,9 @@ import VX_raster_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam PE_IDX_MUFU = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; +`endif VX_execute_if #( .data_t (sfu_execute_t) @@ -146,6 +149,11 @@ import VX_raster_pkg::*; pe_select = PE_SEL_BITS'(PE_IDX_RTUW); end `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + if (per_block_execute_if[0].data.op_type == INST_SFU_MUFU) begin + pe_select = PE_SEL_BITS'(PE_IDX_MUFU); + end + `endif end VX_pe_switch #( @@ -291,6 +299,18 @@ import VX_raster_pkg::*; ); `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + VX_mufu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) mufu_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MUFU]), + .result_if (pe_result_if[PE_IDX_MUFU]) + ); +`endif + VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), diff --git a/hw/rtl/fpu/VX_fcvt_unit.sv b/hw/rtl/fpu/VX_fcvt_unit.sv index e0ee4ed553..c14bdbcc4e 100644 --- a/hw/rtl/fpu/VX_fcvt_unit.sv +++ b/hw/rtl/fpu/VX_fcvt_unit.sv @@ -99,20 +99,33 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [15:0] h_op = (&dataa[`VX_CFG_XLEN-1:16]) ? dataa[15:0] : 16'h7E00; + wire [31:0] s_op; + if (HAS_D) begin : g_s_unbox + assign s_op = (&dataa[`VX_CFG_XLEN-1:32]) ? dataa[31:0] : 32'h7FC00000; + end else begin : g_s_raw + assign s_op = dataa[31:0]; + end + // Float source field extraction (per source format) - wire input_fp_sgn = src_is_h ? h_op[15] : src_is_d ? safe_dataa[63] : safe_dataa[31]; + wire input_fp_sgn = src_is_h ? h_op[15] : src_is_d ? safe_dataa[63] : s_op[31]; fclass_t fclass32; - VX_fp_classifier #(.EXP_BITS(F32_EXP), .MAN_BITS(F32_MAN)) fp_classifier32 ( - .exp_i (safe_dataa[F32_MAN +: F32_EXP]), - .man_i (safe_dataa[F32_MAN-1:0]), + VX_fp_classifier #( + .EXP_BITS (F32_EXP), + .MAN_BITS (F32_MAN) + ) fp_classifier32 ( + .exp_i (s_op[F32_MAN +: F32_EXP]), + .man_i (s_op[F32_MAN-1:0]), .clss_o (fclass32) ); fclass_t fclass_sd; if (HAS_D) begin : g_classify_d fclass_t fclass64; - VX_fp_classifier #(.EXP_BITS(F64_EXP), .MAN_BITS(F64_MAN)) fp_classifier64 ( + VX_fp_classifier #( + .EXP_BITS (F64_EXP), + .MAN_BITS (F64_MAN) + ) fp_classifier64 ( .exp_i (safe_dataa[F64_MAN +: F64_EXP]), .man_i (safe_dataa[F64_MAN-1:0]), .clss_o (fclass64) @@ -125,7 +138,10 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; fclass_t fclass; if (HAS_H) begin : g_classify_h fclass_t fclass16; - VX_fp_classifier #(.EXP_BITS(F16_EXP), .MAN_BITS(F16_MAN)) fp_classifier16 ( + VX_fp_classifier #( + .EXP_BITS (F16_EXP), + .MAN_BITS (F16_MAN) + ) fp_classifier16 ( .exp_i (h_op[F16_MAN +: F16_EXP]), .man_i (h_op[F16_MAN-1:0]), .clss_o (fclass16) @@ -144,16 +160,16 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [S_MAN_WIDTH-1:0] fp_unpacked_mant = src_daz ? '0 : src_is_h ? S_MAN_WIDTH'({fclass.is_normal, h_op[F16_MAN-1:0]}) : src_is_d ? S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F64_MAN-1:0]}) - : S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F32_MAN-1:0]}); + : S_MAN_WIDTH'({fclass.is_normal, s_op[F32_MAN-1:0]}); wire signed [S_EXP_WIDTH-1:0] src_bias_s = src_is_h ? S_EXP_WIDTH'(F16_BIAS) : src_is_d ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); wire [SUPER_EXP-1:0] src_exp_sd; if (HAS_D) begin : g_srcexp_d assign src_exp_sd = src_is_d ? safe_dataa[F64_MAN +: F64_EXP] - : SUPER_EXP'(safe_dataa[F32_MAN +: F32_EXP]); + : SUPER_EXP'(s_op[F32_MAN +: F32_EXP]); end else begin : g_srcexp_s - assign src_exp_sd = safe_dataa[F32_MAN +: F32_EXP]; + assign src_exp_sd = s_op[F32_MAN +: F32_EXP]; end wire [SUPER_EXP-1:0] src_exp_raw = src_is_h ? SUPER_EXP'(h_op[F16_MAN +: F16_EXP]) : src_exp_sd; @@ -546,17 +562,18 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; reg [`VX_CFG_XLEN-1:0] dst_r; always_comb begin - if (is_nan_src) + if (is_nan_src) begin dst_r = dst_d_s5 ? qnan_d : dst_h_s5 ? box_h_qnan : box_s_qnan; - else if (is_inf_src) + end else if (is_inf_src) begin dst_r = dst_d_s5 ? inf_d : dst_h_s5 ? box_h_inf : box_s_inf; - else if (is_zero_src) + end else if (is_zero_src) begin dst_r = dst_d_s5 ? zero_d : dst_h_s5 ? box_h_zero : box_s_zero; - else if (f2f_of_s5) + end else if (f2f_of_s5) begin dst_r = dst_h_s5 ? (of_to_inf ? box_h_inf : box_h_max) : (of_to_inf ? box_s_inf : box_s_max); - else + end else begin dst_r = normal_dst; + end end assign fp_dst_res = dst_r; assign f2f_nv_s5 = is_f2f_s5 & fclass_s5.is_signaling; diff --git a/hw/rtl/libs/VX_rom.sv b/hw/rtl/libs/VX_rom.sv new file mode 100644 index 0000000000..424d81a7a1 --- /dev/null +++ b/hw/rtl/libs/VX_rom.sv @@ -0,0 +1,58 @@ +`include "VX_platform.vh" + +`TRACING_OFF +module VX_rom #( + parameter DATAW = 1, + parameter SIZE = 1, + parameter OUT_REG = 0, + parameter [SIZE*DATAW-1:0] INIT_DATA = '0, + parameter ADDRW = `LOG2UP(SIZE) +) ( + input wire clk, + input wire read, + input wire [ADDRW-1:0] addr, + output wire [DATAW-1:0] rdata +); +`ifdef SYNTHESIS + localparam BLOCK = (OUT_REG != 0) && `FORCE_BRAM(SIZE, DATAW); +`else + localparam BLOCK = 0; +`endif + + if (BLOCK) begin : g_block + `USE_BLOCK_ROM reg [DATAW-1:0] mem [0:SIZE-1]; + reg [DATAW-1:0] rdata_r; + initial begin + for (integer i = 0; i < SIZE; ++i) begin + mem[i] = INIT_DATA[i * DATAW +: DATAW]; + end + end + always @(posedge clk) begin + if (read) begin + rdata_r <= mem[addr]; + end + end + assign rdata = rdata_r; + end else begin : g_logic + reg [DATAW-1:0] mem [0:SIZE-1]; + initial begin + for (integer i = 0; i < SIZE; ++i) begin + mem[i] = INIT_DATA[i * DATAW +: DATAW]; + end + end + if (OUT_REG != 0) begin : g_out_reg + reg [DATAW-1:0] rdata_r; + always @(posedge clk) begin + if (read) begin + rdata_r <= mem[addr]; + end + end + assign rdata = rdata_r; + end else begin : g_no_out_reg + `UNUSED_VAR ({clk, read}) + assign rdata = mem[addr]; + end + end + +endmodule +`TRACING_ON diff --git a/hw/rtl/mufu/VX_mufu_pe.sv b/hw/rtl/mufu/VX_mufu_pe.sv index 24619244be..3a57f48ac3 100644 --- a/hw/rtl/mufu/VX_mufu_pe.sv +++ b/hw/rtl/mufu/VX_mufu_pe.sv @@ -1,8 +1,6 @@ `include "VX_define.vh" -module VX_mufu_pe import VX_gpu_pkg::*; #( - parameter LATENCY = 7 -) ( +module VX_mufu_pe import VX_gpu_pkg::*; ( input wire clk, input wire reset, input wire enable, @@ -10,7 +8,7 @@ module VX_mufu_pe import VX_gpu_pkg::*; #( input wire [31:0] dataa, output wire [31:0] result ); - `STATIC_ASSERT (LATENCY == 7, ("VX_mufu_pe: LATENCY must be 7, got %0d", LATENCY)) + `STATIC_ASSERT (`VX_MUFU_LATENCY == 7, ("VX_mufu_pe: the datapath is 7 stages, VX_MUFU_LATENCY=%0d", `VX_MUFU_LATENCY)) localparam TBL_RCP = 3'd0; localparam TBL_RSQRT = 3'd1; diff --git a/hw/rtl/mufu/VX_mufu_rom.sv b/hw/rtl/mufu/VX_mufu_rom.sv index 28b86a76f1..6a3120288a 100644 --- a/hw/rtl/mufu/VX_mufu_rom.sv +++ b/hw/rtl/mufu/VX_mufu_rom.sv @@ -9,548 +9,564 @@ module VX_mufu_rom ( output wire signed [26:0] a1, output wire signed [19:0] a2 ); + localparam DATAW = 72; + localparam SIZE = 512; + + localparam [SIZE*DATAW-1:0] ROM_DATA = { + 72'h01fffff0000017fffe, + 72'h01ffffc800003ffffc, + 72'h01ffff68000097fff4, + 72'h01fffe68000187ffe0, + 72'h01fffba800041fffa9, + 72'h01fff420000b2fff13, + 72'h01ffdfc0001e57fd7c, + 72'h01ffa86000525ff92a, + 72'h03fede58008e17f8e9, + 72'h03fe229800ea17f453, + 72'h03fced6001817fecc8, + 72'h03faf050027a5fe06a, + 72'h03f7ab280412afcc31, + 72'h03f24cc006ae6fab4b, + 72'h03e982780aecbf7643, + 72'h03db2e7011c4d72203, + 72'h07a1e40016e18f6344, + 72'h0787f1c01d014f3b68, + 72'h07671e2824a3770b06, + 72'h073dc1082e1336d146, + 72'h0709e5a8399e3e8de0, + 72'h06c94c50478cb64190, + 72'h0679767058143deeb1, + 72'h0617c2406b453d99fc, + 72'h0bbec2a075fefeaec1, + 72'h0b433350811b669c9c, + 72'h0abc48a08cb58e8cf6, + 72'h0a29927898b26e80b0, + 72'h098abf50a4ef9e78ba, + 72'h08dfa408b1430e7603, + 72'h08284348bd7b0e7982, + 72'h0764d478c95fce83fb, + 72'h0dfd7340cf15af4856, + 72'h0d2b94d0d4a7774ddb, + 72'h0c543ee0da04f75465, + 72'h0b77a9b8df25ff5bf3, + 72'h0a9615e8e402776485, + 72'h09afcc00e8925f6e17, + 72'h08c51c78eccdf778a0, + 72'h07d65f48f0add78416, + 72'h0ebb32e0f2723fc69f, + 72'h0dc7e820f423dfc9ce, + 72'h0cd2f8d8f5bb37cd17, + 72'h0bdc7fc0f7378fd077, + 72'h0ae49838f89827d3ee, + 72'h09eb5e38f9dc67d77a, + 72'h08f0ee78fb03afdb19, + 72'h07f56640fc0d67deca, + 72'h0eee84b0fc84e7f0cd, + 72'h0df1c6d8fcf6f7f1bf, + 72'h0cf49ac8fd615ff2b3, + 72'h0bf70828fdc41ff3a8, + 72'h0af916a8fe1f1ff4a0, + 72'h09face00fe725ff598, + 72'h08fc3600febdd7f692, + 72'h07fd5680ff016ff78d, + 72'h0efb9cb0ff1fb7fc25, + 72'h0dfc6e90ff3c97fc64, + 72'h0cfd2490ff577ffca3, + 72'h0bfdc0a0ff706ffce3, + 72'h0afe44c8ff875ffd22, + 72'h09feb300ff9c4ffd62, + 72'h08ff0d40ffaf47fda1, + 72'h07ff5588ffc03ffde1, + 72'h7ff6218013bced8867, + 72'h7fd887882776e588c9, + 72'h7fa736b03b2ac5898b, + 72'h7f6236884ed58d8aaf, + 72'h7f0991c06274258c34, + 72'h7e9d55f07603958e1a, + 72'h7e1d93e08980cd9060, + 72'h7d8a5f389ce8cd9307, + 72'h7ce3cea0b038a5960d, + 72'h7c29fbe0c36d4d9973, + 72'h7b5d0390d683cd9d37, + 72'h7a7d0548e9793da159, + 72'h798a23a0fc4aada5d9, + 72'h788484010ef535aab6, + 72'h776c4ec92175f5afef, + 72'h7641af2933ca0db584, + 72'h7504d33145eeb5bb72, + 72'h73b5ebb157e11dc1bb, + 72'h72552c69699e75c85b, + 72'h70e2cba97b240dcf54, + 72'h6f5f02918c6f2dd6a3, + 72'h6dca0cf19d7d2dde47, + 72'h6c242941ae4b6de63f, + 72'h6a6d9881bed74dee8a, + 72'h68a69e61cf1e45f727, + 72'h66cf8101df1dd60014, + 72'h64e88901eed3860950, + 72'h62f20189fe3ce612da, + 72'h60ec38020d579e1cb0, + 72'h5ed77c621c214626d0, + 72'h5cb420b22a97ae3139, + 72'h5a82797238b88e3bea, + 72'h5842dd224681be46e0, + 72'h55f5a4a253f116521a, + 72'h539b2ac26104865d97, + 72'h5133cc626dba0e6954, + 72'h4ebfe8727a0fbe7550, + 72'h4c3fdfc286039e8188, + 72'h49b415029193e68dfc, + 72'h471cecb29cbec69aa9, + 72'h447acd22a7827ea78c, + 72'h41ce1e32b1dd76b4a5, + 72'h3f174982bbce0ec1f1, + 72'h3c56ba3ac552b6cf6e, + 72'h398cdd02ce69fedd1a, + 72'h36ba1fdad71286eaf3, + 72'h33def252df4ae6f8f7, + 72'h30fbc512e711e70723, + 72'h2e110a2aee664f1576, + 72'h2b1f34b2f5470723ec, + 72'h2826b8f2fbb2ef3285, + 72'h25280c2301a917413d, + 72'h2223a48b07288f5013, + 72'h1f19f93b0c30875f04, + 72'h1c0b822b10c0276e0e, + 72'h18f8b7fb14d6c77d2e, + 72'h15e214031873bf8c62, + 72'h12c810331b96879ba8, + 72'h0fab26eb1e3e9faafe, + 72'h0c8bd323206b9fba61, + 72'h096a900b221d2fc9ce, + 72'h0647d94323530fd944, + 72'h03242a83240d0fe8c0, + 72'h00000003244857f8cf, + 72'h7e8d3859743bdfa2f1, + 72'h7d17824177301fa177, + 72'h7b9ed1d97a307f9ff4, + 72'h7a231ae17d3d3f9e67, + 72'h78a450c98056b79cd1, + 72'h772266c1837d2f9b30, + 72'h759d4f9186b0f79986, + 72'h7414fdc989f26f97d1, + 72'h728963898d41df9610, + 72'h70fa72a1909fb79445, + 72'h6f681c91940c3f926d, + 72'h6dd252519787ef908a, + 72'h6c3904b19b13178e99, + 72'h6a9c23f19eae2f8c9c, + 72'h68fb9fe9a2599f8a91, + 72'h67576811a615cf8878, + 72'h65af6b61a9e33f8651, + 72'h64039871adc25f841b, + 72'h6253dd49b1b3af81d5, + 72'h60a02771b5b7af7f7f, + 72'h5ee86401b9cee77d18, + 72'h5d2c7f79bdf9df7a9f, + 72'h5b6c65c9c239277815, + 72'h59a80259c68d5f7578, + 72'h57df3ff1caf71772c7, + 72'h561208c1cf76f77002, + 72'h54404641d40da76d29, + 72'h5269e151d8bbcf6a39, + 72'h508ec221dd822f6732, + 72'h4eaed011e261876414, + 72'h4cc9f1d1e75a8f60dd, + 72'h4ae00d41ec6e1f5d8c, + 72'h48f10779f19d075a21, + 72'h46fcc4a1f6e82f5699, + 72'h45032811fc507752f4, + 72'h4304143201d6d74f31, + 72'h40ff6a5a077c4f4b4f, + 72'h3ef50b020d41df474b, + 72'h3ce4d5721328a74324, + 72'h3acea7f21931c73ed9, + 72'h38b25f921f5e673a69, + 72'h368fd82225afcf35d0, + 72'h3466ec522c273f310e, + 72'h3237755232c6172c21, + 72'h30014b02398dc72706, + 72'h2dc443da407fcf21bb, + 72'h2b8034b2479dbf1c3f, + 72'h2934f0da4ee937168d, + 72'h26e249e25663ef10a4, + 72'h24880f9a5e0fc70a81, + 72'h2226100265ee970420, + 72'h1fbc17026e0276fd80, + 72'h1d49ee9a764d6ef69b, + 72'h1acf5e827ed1ceef6f, + 72'h184c2c228791e6e7f7, + 72'h15c01a92909036e030, + 72'h132aea4299cf5ed814, + 72'h108c58eaa35226cfa0, + 72'h0de42182ad1b7ec6cd, + 72'h0b31fbe2b72e86bd97, + 72'h08759cbac18e86b3f7, + 72'h05aeb552cc3efea9e8, + 72'h02dcf34ad743a69f61, + 72'h00000002e2a27693f4, + 72'hfd3e0c82be2110f4aa, + 72'hfa83b352b690d8f207, + 72'hf7d0dfeaaf1580ef6c, + 72'hf5257d8aa7aec8ecd8, + 72'hf28177b2a05c78ea4b, + 72'hefe4ba12991e58e7c4, + 72'hed4f309291f430e545, + 72'heac0c75a8addd0e2cd, + 72'he8396ac283daf8e05c, + 72'he5b907527ceb70ddf1, + 72'he33f89e2760f10db8d, + 72'he0ccdf5a6f4598d92f, + 72'hde60f4ea688ed8d6d9, + 72'hdbfbb80261ea98d488, + 72'hd99d162a5b58b0d23e, + 72'hd744fd3254d8e0cffa, + 72'hd4f35b124e6b00cdbd, + 72'hd2a81df2480ed8cb85, + 72'hd063344241c438c954, + 72'hce248c7a3b8af0c729, + 72'hcbec15623562d8c504, + 72'hc9b9bdea2f4bb8c2e4, + 72'hc78d752a294568c0cb, + 72'hc5672a72234fb0beb7, + 72'hc346cd3a1d6a68bca9, + 72'hc12c4d2a179560baa1, + 72'hbf179a1211d070b89f, + 72'hbd08a3fa0c1b68b6a1, + 72'hbaff5b0a067620b4aa, + 72'hb8fbafa200e060b2b8, + 72'hb6fd9239fb5a10b0cb, + 72'hb504f389f5e2f8aee3, + 72'hb311c469f07af8ad01, + 72'hb123f5d9eb21d8ab24, + 72'haf3b7901e5d780a94c, + 72'had583f41e09bb8a779, + 72'hab7a3a09db6e60a5ab, + 72'ha9a15b09d64f50a3e2, + 72'ha7cd9409d13e60a21e, + 72'ha5fed6f9cc3b68a05f, + 72'ha4351601c746409ea5, + 72'ha2704351c25ec89cf0, + 72'ha0b05161bd84d09b3f, + 72'h9ef532b1b8b8409993, + 72'h9d3ed9f1b3f8e897eb, + 72'h9b8d3a01af46a89648, + 72'h99e045d9aaa16094aa, + 72'h9837f099a608e09310, + 72'h96942d81a17d10917a, + 72'h94f4eff19cfdc88fe9, + 72'h935a2b79988ae88e5c, + 72'h91c3d3b99424488cd4, + 72'h9031dc898fc9d08b4f, + 72'h8ea439d18b7b5889cf, + 72'h8d1adfa18738c08853, + 72'h8b95c2298301e886db, + 72'h8a14d5b97ed6b08567, + 72'h88980ec17ab6f883f8, + 72'h871f61d976a2a0828c, + 72'h85aac3a97299808124, + 72'h843a29016e9b887fc0, + 72'h82cd86d96aa8907e5f, + 72'h8164d23166c0807d03, + 72'h8000000162e3687b9e, + 72'h404060a7bf3e78183c, + 72'h40818517be78b818b8, + 72'h40c3713fbdaf081937, + 72'h41062927bce14019b9, + 72'h4149b0e7bc0f481a3f, + 72'h418e0ccfbb39001ac9, + 72'h41d34127ba5e501b57, + 72'h4219528fb97f081be9, + 72'h4260458fb89b101c80, + 72'h42a81ef7b7b2381d1b, + 72'h42f0e3afb6c4701dba, + 72'h433a98c7b5d1801e5e, + 72'h43854377b4d9401f08, + 72'h43d0e917b3db901fb6, + 72'h441d8f3fb2d840206a, + 72'h446b3b97b1cf282124, + 72'h44b9f407b0c01021e4, + 72'h4509beafafaac822a9, + 72'h455aa1cfae8f202375, + 72'h45aca3d7ad6ce82448, + 72'h45ffcb7fac43e02521, + 72'h46541fb7ab13d82602, + 72'h46a9a797a9dc8826ea, + 72'h47006a7fa89dc027da, + 72'h47587017a7573028d2, + 72'h47b1c047a608a029d3, + 72'h480c632fa4b1b82add, + 72'h48686147a352402bf0, + 72'h48c5c347a1e9e02d0d, + 72'h49249247a078502e34, + 72'h4984d79f9efd282f65, + 72'h49e69d179d782030a2, + 72'h4a49ecaf9be8d831ea, + 72'h4aaed0ef9a4ee0333f, + 72'h4b15549f98a9e834a1, + 72'h4b7d830f96f9703610, + 72'h4be767ef953d10378d, + 72'h4c530f579374503919, + 72'h4cc085f7919eb03ab5, + 72'h4d2fd8e78fbbb03c61, + 72'h4da115cf8dcac83e1f, + 72'h4e144adf8bcb603fee, + 72'h4e8986d789bce841d1, + 72'h4f00d937879eb043c8, + 72'h4f7a51f785702045d4, + 72'h4ff601cf83307847f7, + 72'h5073fa4780def84a31, + 72'h50f44d777e7ae84c85, + 72'h51770e7f7c03684ef2, + 72'h51fc512f797798517c, + 72'h52842a4776d6905424, + 72'h530eaf8f741f5056ea, + 72'h539bf7b77150d059d3, + 72'h542c1a8f6e69f05cde, + 72'h54bf31076b6990600f, + 72'h55555537684e706368, + 72'h55eea2a765173066eb, + 72'h568b360f61c2786a9b, + 72'h572b2dbf5e4eb86e7c, + 72'h57cea9775aba60728f, + 72'h5875cab75703b876da, + 72'h5920b4b75328e87b5e, + 72'h59cf8c8f4f28088022, + 72'h5a82796f4afef08528, + 72'h5add84bfa46bd02246, + 72'h5b39a4c7a3543022f5, + 72'h5b96df47a236f023a8, + 72'h5bf539e7a113f02461, + 72'h5c54ba7f9feb00251e, + 72'h5cb567179ebbf825e1, + 72'h5d1745d79d86a826aa, + 72'h5d7a5d179c4ae82779, + 72'h5ddeb3779b0880284e, + 72'h5e444faf99bf382929, + 72'h5eab38a7986ef02a0a, + 72'h5f1375979717582af3, + 72'h5f7d0dd795b8502be2, + 72'h5fe808ff9451882cd9, + 72'h60546edf92e2d02dd8, + 72'h60c24797916be82ede, + 72'h61319b778fec802fed, + 72'h61a2731f8e64603105, + 72'h6214d75f8cd3403225, + 72'h6288d16f8b38d0334f, + 72'h62fe6abf8994c03482, + 72'h6375ad0f87e6c835c0, + 72'h63eea27f862e883709, + 72'h6469557f846bb0385c, + 72'h64e5d0d7829de039bb, + 72'h65641fa780c4b03b26, + 72'h65e44d877edfc83c9e, + 72'h6666665f7ceeb83e23, + 72'h66ea768f7af1183fb6, + 72'h67708aef78e6684157, + 72'h67f8b0b776ce404307, + 72'h6882f5b774a81044c7, + 72'h690f681f7273604697, + 72'h699e16c7702fa04879, + 72'h6a2f10f76ddc404a6d, + 72'h6ac266a76b78a84c75, + 72'h6b5828676904384e90, + 72'h6bf06757667e4050c0, + 72'h6c8b354763e6205306, + 72'h6d28a4df613b105564, + 72'h6dc8c95f5e7c5057da, + 72'h6e6bb6d75ba9185a69, + 72'h6f11823758c0885d14, + 72'h6fba414755c1c05fdb, + 72'h70660aaf52abd062c1, + 72'h7114f62f4f7db065c6, + 72'h71c71c574c366868ec, + 72'h727c96ff48d4c86c36, + 72'h733580ff4557b06fa6, + 72'h73f1f66f41bde8733d, + 72'h74b214af3e061876fe, + 72'h7575fa873a2ed87aeb, + 72'h763dc7ff3636c07f07, + 72'h77099ed7321c308356, + 72'h77d9a2472ddd9087d9, + 72'h78adf74f2979088c95, + 72'h7986c4b724ecd0918c, + 72'h7a6433372036e096c4, + 72'h7b466da71b55109c3f, + 72'h7c2da0ef164528a203, + 72'h7d19fc671104b8a814, + 72'h7e0bb1e70b9128ae78, + 72'h7f02f5df05e7c0b535, + 72'h80000007000438bc93, + 72'h40404047bf7f601018, + 72'h40810207befd201049, + 72'h40c246dfbe7948107a, + 72'h41041047bdf3e810ad, + 72'h41465fe7bd6ce810e0, + 72'h41893757bce4481114, + 72'h41cc982fbc5a081149, + 72'h42108427bbce10117e, + 72'h4254fce7bb407011b5, + 72'h429a042fbab11011ec, + 72'h42df9bb7ba1ff01224, + 72'h4325c547b98d00125e, + 72'h436c82a7b8f8401298, + 72'h43b3d5b7b861b012d3, + 72'h43fbc047b7c938130f, + 72'h44444447b72ed8134c, + 72'h448d63a7b69280138a, + 72'h44d72047b5f43813c9, + 72'h45217c3fb553e8140a, + 72'h456c7987b4b190144b, + 72'h45b81a27b40d28148e, + 72'h4604604fb366a014d1, + 72'h46514e07b2bdf01516, + 72'h469ee587b21318155c, + 72'h46ed2907b1660015a3, + 72'h473c1abfb0b6a015eb, + 72'h478bbcf7b004f81635, + 72'h47dc11f7af51001680, + 72'h482d1c37ae9a9816cc, + 72'h487ede0fade1c8171a, + 72'h48d159e7ad26881769, + 72'h4924924fac68b817b9, + 72'h497889c7aba860180b, + 72'h49cd42e7aae570185f, + 72'h4a22c04faa1fd018b4, + 72'h4a7904afa95780190a, + 72'h4ad012b7a88c701962, + 72'h4b27ed37a7be9819bc, + 72'h4b809707a6ede01a17, + 72'h4bda12ffa61a401a74, + 72'h4c346407a543b01ad3, + 72'h4c8f8d2fa46a181b33, + 72'h4ceb916fa38d701b95, + 72'h4d4873f7a2ada01bfa, + 72'h4da637d7a1caa81c60, + 72'h4e04e057a0e4681cc8, + 72'h4e6470b79ffae01d32, + 72'h4ec4ec4f9f0df81d9e, + 72'h4f2656979e1d981e0c, + 72'h4f88b2f79d29b81e7c, + 72'h4fec04ff9c32481eef, + 72'h505050579b37281f63, + 72'h50b5989f9a38581fda, + 72'h511be1979935c02054, + 72'h51832f27982f4020d0, + 72'h51eb851f9724d8214e, + 72'h5254e78f96166021cf, + 72'h52bf5a879503d02252, + 72'h532ae21f93ed1022d8, + 72'h5397829f92d2082361, + 72'h5405405791b2a023ed, + 72'h54741faf908ec8247c, + 72'h54e425278f6660250d, + 72'h555555578e395825a2, + 72'h55c7b4f78d07882639, + 72'h563b48c78bd0e826d4, + 72'h56b015af8a95582772, + 72'h572620b78954b82814, + 72'h579d6ee7880ef828b9, + 72'h5816058786c3e82962, + 72'h588fe9d78573802a0e, + 72'h590b2167841d902abe, + 72'h5987b1a782c2002b72, + 72'h5a05a0578160b82c2a, + 72'h5a84f3477ff9902ce6, + 72'h5b05b0577e8c682da6, + 72'h5b87ddaf7d19182e6a, + 72'h5c0b816f7b9f802f33, + 72'h5c90a1ff7a1f803000, + 72'h5d1745cf7898f830d2, + 72'h5d9f738f770bb031a9, + 72'h5e2932077577883285, + 72'h5eb4881f73dc603366, + 72'h5f417d07723a00344c, + 72'h5fd017ef7090483538, + 72'h6060605f6edf083629, + 72'h60f25de76d26103720, + 72'h6186185f6b6530381c, + 72'h621b97bf699c38391f, + 72'h62b2e43f67caf83a28, + 72'h634c063765f1403b38, + 72'h63e70637640ed83c4e, + 72'h6483ed276223803d6b, + 72'h6522c3ef602f103e8f, + 72'h65c393df5e31403fba, + 72'h666666675c29e040ed, + 72'h670b45375a18a84228, + 72'h67b23a4f57fd58436a, + 72'h685b4fdf55d7b844b5, + 72'h6906906753a7784608, + 72'h69b40697516c584765, + 72'h6a63bd774f261048ca, + 72'h6b15c0674cd4584a38, + 72'h6bca1ae74a76d84bb1, + 72'h6c80d8f7480d484d33, + 72'h6d3a06cf4597504ebf, + 72'h6df5b0e74314a85057, + 72'h6eb3e4474084e851f9, + 72'h6f74ae173de7b853a7, + 72'h70381c073b3cc05560, + 72'h70fe3bf73883985726, + 72'h71c71c6735bbd858f8, + 72'h7292cc0732e5285ad8, + 72'h73615a172fff085cc5, + 72'h7432d6372d09105ec0, + 72'h750750672a02d060ca, + 72'h75ded94726ebc862e2, + 72'h76b981c723c380650b, + 72'h77975b7f2089706743, + 72'h787878671d3d18698c, + 72'h795ceb0f19ddf06be7, + 72'h7a44c69f166b586e54, + 72'h7b301eb712e4d070d3, + 72'h7c1f07af0f49a87366, + 72'h7d1196670b9950760d, + 72'h7e07e06707d31078c9, + 72'h7f01fbef03f6487b9b, + 72'h800000070001b07e9f + }; reg [8:0] base; always @(*) begin case (tbl) - 3'd0: base = 9'd0; - 3'd1: base = 9'd128; - 3'd2: base = 9'd256; - 3'd3: base = 9'd320; - 3'd4: base = 9'd384; - 3'd5: base = 9'd448; - default: base = '0; + 3'd0: base = 9'd0; + 3'd1: base = 9'd128; + 3'd2: base = 9'd256; + 3'd3: base = 9'd320; + 3'd4: base = 9'd384; + 3'd5: base = 9'd448; + default: base = '0; endcase end wire [8:0] addr = base + 9'(idx); - (* rom_style = "block" *) reg [71:0] mem [0:511]; - - initial begin - mem[0] = 72'h800000070001b07e9f; - mem[1] = 72'h7f01fbef03f6487b9b; - mem[2] = 72'h7e07e06707d31078c9; - mem[3] = 72'h7d1196670b9950760d; - mem[4] = 72'h7c1f07af0f49a87366; - mem[5] = 72'h7b301eb712e4d070d3; - mem[6] = 72'h7a44c69f166b586e54; - mem[7] = 72'h795ceb0f19ddf06be7; - mem[8] = 72'h787878671d3d18698c; - mem[9] = 72'h77975b7f2089706743; - mem[10] = 72'h76b981c723c380650b; - mem[11] = 72'h75ded94726ebc862e2; - mem[12] = 72'h750750672a02d060ca; - mem[13] = 72'h7432d6372d09105ec0; - mem[14] = 72'h73615a172fff085cc5; - mem[15] = 72'h7292cc0732e5285ad8; - mem[16] = 72'h71c71c6735bbd858f8; - mem[17] = 72'h70fe3bf73883985726; - mem[18] = 72'h70381c073b3cc05560; - mem[19] = 72'h6f74ae173de7b853a7; - mem[20] = 72'h6eb3e4474084e851f9; - mem[21] = 72'h6df5b0e74314a85057; - mem[22] = 72'h6d3a06cf4597504ebf; - mem[23] = 72'h6c80d8f7480d484d33; - mem[24] = 72'h6bca1ae74a76d84bb1; - mem[25] = 72'h6b15c0674cd4584a38; - mem[26] = 72'h6a63bd774f261048ca; - mem[27] = 72'h69b40697516c584765; - mem[28] = 72'h6906906753a7784608; - mem[29] = 72'h685b4fdf55d7b844b5; - mem[30] = 72'h67b23a4f57fd58436a; - mem[31] = 72'h670b45375a18a84228; - mem[32] = 72'h666666675c29e040ed; - mem[33] = 72'h65c393df5e31403fba; - mem[34] = 72'h6522c3ef602f103e8f; - mem[35] = 72'h6483ed276223803d6b; - mem[36] = 72'h63e70637640ed83c4e; - mem[37] = 72'h634c063765f1403b38; - mem[38] = 72'h62b2e43f67caf83a28; - mem[39] = 72'h621b97bf699c38391f; - mem[40] = 72'h6186185f6b6530381c; - mem[41] = 72'h60f25de76d26103720; - mem[42] = 72'h6060605f6edf083629; - mem[43] = 72'h5fd017ef7090483538; - mem[44] = 72'h5f417d07723a00344c; - mem[45] = 72'h5eb4881f73dc603366; - mem[46] = 72'h5e2932077577883285; - mem[47] = 72'h5d9f738f770bb031a9; - mem[48] = 72'h5d1745cf7898f830d2; - mem[49] = 72'h5c90a1ff7a1f803000; - mem[50] = 72'h5c0b816f7b9f802f33; - mem[51] = 72'h5b87ddaf7d19182e6a; - mem[52] = 72'h5b05b0577e8c682da6; - mem[53] = 72'h5a84f3477ff9902ce6; - mem[54] = 72'h5a05a0578160b82c2a; - mem[55] = 72'h5987b1a782c2002b72; - mem[56] = 72'h590b2167841d902abe; - mem[57] = 72'h588fe9d78573802a0e; - mem[58] = 72'h5816058786c3e82962; - mem[59] = 72'h579d6ee7880ef828b9; - mem[60] = 72'h572620b78954b82814; - mem[61] = 72'h56b015af8a95582772; - mem[62] = 72'h563b48c78bd0e826d4; - mem[63] = 72'h55c7b4f78d07882639; - mem[64] = 72'h555555578e395825a2; - mem[65] = 72'h54e425278f6660250d; - mem[66] = 72'h54741faf908ec8247c; - mem[67] = 72'h5405405791b2a023ed; - mem[68] = 72'h5397829f92d2082361; - mem[69] = 72'h532ae21f93ed1022d8; - mem[70] = 72'h52bf5a879503d02252; - mem[71] = 72'h5254e78f96166021cf; - mem[72] = 72'h51eb851f9724d8214e; - mem[73] = 72'h51832f27982f4020d0; - mem[74] = 72'h511be1979935c02054; - mem[75] = 72'h50b5989f9a38581fda; - mem[76] = 72'h505050579b37281f63; - mem[77] = 72'h4fec04ff9c32481eef; - mem[78] = 72'h4f88b2f79d29b81e7c; - mem[79] = 72'h4f2656979e1d981e0c; - mem[80] = 72'h4ec4ec4f9f0df81d9e; - mem[81] = 72'h4e6470b79ffae01d32; - mem[82] = 72'h4e04e057a0e4681cc8; - mem[83] = 72'h4da637d7a1caa81c60; - mem[84] = 72'h4d4873f7a2ada01bfa; - mem[85] = 72'h4ceb916fa38d701b95; - mem[86] = 72'h4c8f8d2fa46a181b33; - mem[87] = 72'h4c346407a543b01ad3; - mem[88] = 72'h4bda12ffa61a401a74; - mem[89] = 72'h4b809707a6ede01a17; - mem[90] = 72'h4b27ed37a7be9819bc; - mem[91] = 72'h4ad012b7a88c701962; - mem[92] = 72'h4a7904afa95780190a; - mem[93] = 72'h4a22c04faa1fd018b4; - mem[94] = 72'h49cd42e7aae570185f; - mem[95] = 72'h497889c7aba860180b; - mem[96] = 72'h4924924fac68b817b9; - mem[97] = 72'h48d159e7ad26881769; - mem[98] = 72'h487ede0fade1c8171a; - mem[99] = 72'h482d1c37ae9a9816cc; - mem[100] = 72'h47dc11f7af51001680; - mem[101] = 72'h478bbcf7b004f81635; - mem[102] = 72'h473c1abfb0b6a015eb; - mem[103] = 72'h46ed2907b1660015a3; - mem[104] = 72'h469ee587b21318155c; - mem[105] = 72'h46514e07b2bdf01516; - mem[106] = 72'h4604604fb366a014d1; - mem[107] = 72'h45b81a27b40d28148e; - mem[108] = 72'h456c7987b4b190144b; - mem[109] = 72'h45217c3fb553e8140a; - mem[110] = 72'h44d72047b5f43813c9; - mem[111] = 72'h448d63a7b69280138a; - mem[112] = 72'h44444447b72ed8134c; - mem[113] = 72'h43fbc047b7c938130f; - mem[114] = 72'h43b3d5b7b861b012d3; - mem[115] = 72'h436c82a7b8f8401298; - mem[116] = 72'h4325c547b98d00125e; - mem[117] = 72'h42df9bb7ba1ff01224; - mem[118] = 72'h429a042fbab11011ec; - mem[119] = 72'h4254fce7bb407011b5; - mem[120] = 72'h42108427bbce10117e; - mem[121] = 72'h41cc982fbc5a081149; - mem[122] = 72'h41893757bce4481114; - mem[123] = 72'h41465fe7bd6ce810e0; - mem[124] = 72'h41041047bdf3e810ad; - mem[125] = 72'h40c246dfbe7948107a; - mem[126] = 72'h40810207befd201049; - mem[127] = 72'h40404047bf7f601018; - mem[128] = 72'h80000007000438bc93; - mem[129] = 72'h7f02f5df05e7c0b535; - mem[130] = 72'h7e0bb1e70b9128ae78; - mem[131] = 72'h7d19fc671104b8a814; - mem[132] = 72'h7c2da0ef164528a203; - mem[133] = 72'h7b466da71b55109c3f; - mem[134] = 72'h7a6433372036e096c4; - mem[135] = 72'h7986c4b724ecd0918c; - mem[136] = 72'h78adf74f2979088c95; - mem[137] = 72'h77d9a2472ddd9087d9; - mem[138] = 72'h77099ed7321c308356; - mem[139] = 72'h763dc7ff3636c07f07; - mem[140] = 72'h7575fa873a2ed87aeb; - mem[141] = 72'h74b214af3e061876fe; - mem[142] = 72'h73f1f66f41bde8733d; - mem[143] = 72'h733580ff4557b06fa6; - mem[144] = 72'h727c96ff48d4c86c36; - mem[145] = 72'h71c71c574c366868ec; - mem[146] = 72'h7114f62f4f7db065c6; - mem[147] = 72'h70660aaf52abd062c1; - mem[148] = 72'h6fba414755c1c05fdb; - mem[149] = 72'h6f11823758c0885d14; - mem[150] = 72'h6e6bb6d75ba9185a69; - mem[151] = 72'h6dc8c95f5e7c5057da; - mem[152] = 72'h6d28a4df613b105564; - mem[153] = 72'h6c8b354763e6205306; - mem[154] = 72'h6bf06757667e4050c0; - mem[155] = 72'h6b5828676904384e90; - mem[156] = 72'h6ac266a76b78a84c75; - mem[157] = 72'h6a2f10f76ddc404a6d; - mem[158] = 72'h699e16c7702fa04879; - mem[159] = 72'h690f681f7273604697; - mem[160] = 72'h6882f5b774a81044c7; - mem[161] = 72'h67f8b0b776ce404307; - mem[162] = 72'h67708aef78e6684157; - mem[163] = 72'h66ea768f7af1183fb6; - mem[164] = 72'h6666665f7ceeb83e23; - mem[165] = 72'h65e44d877edfc83c9e; - mem[166] = 72'h65641fa780c4b03b26; - mem[167] = 72'h64e5d0d7829de039bb; - mem[168] = 72'h6469557f846bb0385c; - mem[169] = 72'h63eea27f862e883709; - mem[170] = 72'h6375ad0f87e6c835c0; - mem[171] = 72'h62fe6abf8994c03482; - mem[172] = 72'h6288d16f8b38d0334f; - mem[173] = 72'h6214d75f8cd3403225; - mem[174] = 72'h61a2731f8e64603105; - mem[175] = 72'h61319b778fec802fed; - mem[176] = 72'h60c24797916be82ede; - mem[177] = 72'h60546edf92e2d02dd8; - mem[178] = 72'h5fe808ff9451882cd9; - mem[179] = 72'h5f7d0dd795b8502be2; - mem[180] = 72'h5f1375979717582af3; - mem[181] = 72'h5eab38a7986ef02a0a; - mem[182] = 72'h5e444faf99bf382929; - mem[183] = 72'h5ddeb3779b0880284e; - mem[184] = 72'h5d7a5d179c4ae82779; - mem[185] = 72'h5d1745d79d86a826aa; - mem[186] = 72'h5cb567179ebbf825e1; - mem[187] = 72'h5c54ba7f9feb00251e; - mem[188] = 72'h5bf539e7a113f02461; - mem[189] = 72'h5b96df47a236f023a8; - mem[190] = 72'h5b39a4c7a3543022f5; - mem[191] = 72'h5add84bfa46bd02246; - mem[192] = 72'h5a82796f4afef08528; - mem[193] = 72'h59cf8c8f4f28088022; - mem[194] = 72'h5920b4b75328e87b5e; - mem[195] = 72'h5875cab75703b876da; - mem[196] = 72'h57cea9775aba60728f; - mem[197] = 72'h572b2dbf5e4eb86e7c; - mem[198] = 72'h568b360f61c2786a9b; - mem[199] = 72'h55eea2a765173066eb; - mem[200] = 72'h55555537684e706368; - mem[201] = 72'h54bf31076b6990600f; - mem[202] = 72'h542c1a8f6e69f05cde; - mem[203] = 72'h539bf7b77150d059d3; - mem[204] = 72'h530eaf8f741f5056ea; - mem[205] = 72'h52842a4776d6905424; - mem[206] = 72'h51fc512f797798517c; - mem[207] = 72'h51770e7f7c03684ef2; - mem[208] = 72'h50f44d777e7ae84c85; - mem[209] = 72'h5073fa4780def84a31; - mem[210] = 72'h4ff601cf83307847f7; - mem[211] = 72'h4f7a51f785702045d4; - mem[212] = 72'h4f00d937879eb043c8; - mem[213] = 72'h4e8986d789bce841d1; - mem[214] = 72'h4e144adf8bcb603fee; - mem[215] = 72'h4da115cf8dcac83e1f; - mem[216] = 72'h4d2fd8e78fbbb03c61; - mem[217] = 72'h4cc085f7919eb03ab5; - mem[218] = 72'h4c530f579374503919; - mem[219] = 72'h4be767ef953d10378d; - mem[220] = 72'h4b7d830f96f9703610; - mem[221] = 72'h4b15549f98a9e834a1; - mem[222] = 72'h4aaed0ef9a4ee0333f; - mem[223] = 72'h4a49ecaf9be8d831ea; - mem[224] = 72'h49e69d179d782030a2; - mem[225] = 72'h4984d79f9efd282f65; - mem[226] = 72'h49249247a078502e34; - mem[227] = 72'h48c5c347a1e9e02d0d; - mem[228] = 72'h48686147a352402bf0; - mem[229] = 72'h480c632fa4b1b82add; - mem[230] = 72'h47b1c047a608a029d3; - mem[231] = 72'h47587017a7573028d2; - mem[232] = 72'h47006a7fa89dc027da; - mem[233] = 72'h46a9a797a9dc8826ea; - mem[234] = 72'h46541fb7ab13d82602; - mem[235] = 72'h45ffcb7fac43e02521; - mem[236] = 72'h45aca3d7ad6ce82448; - mem[237] = 72'h455aa1cfae8f202375; - mem[238] = 72'h4509beafafaac822a9; - mem[239] = 72'h44b9f407b0c01021e4; - mem[240] = 72'h446b3b97b1cf282124; - mem[241] = 72'h441d8f3fb2d840206a; - mem[242] = 72'h43d0e917b3db901fb6; - mem[243] = 72'h43854377b4d9401f08; - mem[244] = 72'h433a98c7b5d1801e5e; - mem[245] = 72'h42f0e3afb6c4701dba; - mem[246] = 72'h42a81ef7b7b2381d1b; - mem[247] = 72'h4260458fb89b101c80; - mem[248] = 72'h4219528fb97f081be9; - mem[249] = 72'h41d34127ba5e501b57; - mem[250] = 72'h418e0ccfbb39001ac9; - mem[251] = 72'h4149b0e7bc0f481a3f; - mem[252] = 72'h41062927bce14019b9; - mem[253] = 72'h40c3713fbdaf081937; - mem[254] = 72'h40818517be78b818b8; - mem[255] = 72'h404060a7bf3e78183c; - mem[256] = 72'h8000000162e3687b9e; - mem[257] = 72'h8164d23166c0807d03; - mem[258] = 72'h82cd86d96aa8907e5f; - mem[259] = 72'h843a29016e9b887fc0; - mem[260] = 72'h85aac3a97299808124; - mem[261] = 72'h871f61d976a2a0828c; - mem[262] = 72'h88980ec17ab6f883f8; - mem[263] = 72'h8a14d5b97ed6b08567; - mem[264] = 72'h8b95c2298301e886db; - mem[265] = 72'h8d1adfa18738c08853; - mem[266] = 72'h8ea439d18b7b5889cf; - mem[267] = 72'h9031dc898fc9d08b4f; - mem[268] = 72'h91c3d3b99424488cd4; - mem[269] = 72'h935a2b79988ae88e5c; - mem[270] = 72'h94f4eff19cfdc88fe9; - mem[271] = 72'h96942d81a17d10917a; - mem[272] = 72'h9837f099a608e09310; - mem[273] = 72'h99e045d9aaa16094aa; - mem[274] = 72'h9b8d3a01af46a89648; - mem[275] = 72'h9d3ed9f1b3f8e897eb; - mem[276] = 72'h9ef532b1b8b8409993; - mem[277] = 72'ha0b05161bd84d09b3f; - mem[278] = 72'ha2704351c25ec89cf0; - mem[279] = 72'ha4351601c746409ea5; - mem[280] = 72'ha5fed6f9cc3b68a05f; - mem[281] = 72'ha7cd9409d13e60a21e; - mem[282] = 72'ha9a15b09d64f50a3e2; - mem[283] = 72'hab7a3a09db6e60a5ab; - mem[284] = 72'had583f41e09bb8a779; - mem[285] = 72'haf3b7901e5d780a94c; - mem[286] = 72'hb123f5d9eb21d8ab24; - mem[287] = 72'hb311c469f07af8ad01; - mem[288] = 72'hb504f389f5e2f8aee3; - mem[289] = 72'hb6fd9239fb5a10b0cb; - mem[290] = 72'hb8fbafa200e060b2b8; - mem[291] = 72'hbaff5b0a067620b4aa; - mem[292] = 72'hbd08a3fa0c1b68b6a1; - mem[293] = 72'hbf179a1211d070b89f; - mem[294] = 72'hc12c4d2a179560baa1; - mem[295] = 72'hc346cd3a1d6a68bca9; - mem[296] = 72'hc5672a72234fb0beb7; - mem[297] = 72'hc78d752a294568c0cb; - mem[298] = 72'hc9b9bdea2f4bb8c2e4; - mem[299] = 72'hcbec15623562d8c504; - mem[300] = 72'hce248c7a3b8af0c729; - mem[301] = 72'hd063344241c438c954; - mem[302] = 72'hd2a81df2480ed8cb85; - mem[303] = 72'hd4f35b124e6b00cdbd; - mem[304] = 72'hd744fd3254d8e0cffa; - mem[305] = 72'hd99d162a5b58b0d23e; - mem[306] = 72'hdbfbb80261ea98d488; - mem[307] = 72'hde60f4ea688ed8d6d9; - mem[308] = 72'he0ccdf5a6f4598d92f; - mem[309] = 72'he33f89e2760f10db8d; - mem[310] = 72'he5b907527ceb70ddf1; - mem[311] = 72'he8396ac283daf8e05c; - mem[312] = 72'heac0c75a8addd0e2cd; - mem[313] = 72'hed4f309291f430e545; - mem[314] = 72'hefe4ba12991e58e7c4; - mem[315] = 72'hf28177b2a05c78ea4b; - mem[316] = 72'hf5257d8aa7aec8ecd8; - mem[317] = 72'hf7d0dfeaaf1580ef6c; - mem[318] = 72'hfa83b352b690d8f207; - mem[319] = 72'hfd3e0c82be2110f4aa; - mem[320] = 72'h00000002e2a27693f4; - mem[321] = 72'h02dcf34ad743a69f61; - mem[322] = 72'h05aeb552cc3efea9e8; - mem[323] = 72'h08759cbac18e86b3f7; - mem[324] = 72'h0b31fbe2b72e86bd97; - mem[325] = 72'h0de42182ad1b7ec6cd; - mem[326] = 72'h108c58eaa35226cfa0; - mem[327] = 72'h132aea4299cf5ed814; - mem[328] = 72'h15c01a92909036e030; - mem[329] = 72'h184c2c228791e6e7f7; - mem[330] = 72'h1acf5e827ed1ceef6f; - mem[331] = 72'h1d49ee9a764d6ef69b; - mem[332] = 72'h1fbc17026e0276fd80; - mem[333] = 72'h2226100265ee970420; - mem[334] = 72'h24880f9a5e0fc70a81; - mem[335] = 72'h26e249e25663ef10a4; - mem[336] = 72'h2934f0da4ee937168d; - mem[337] = 72'h2b8034b2479dbf1c3f; - mem[338] = 72'h2dc443da407fcf21bb; - mem[339] = 72'h30014b02398dc72706; - mem[340] = 72'h3237755232c6172c21; - mem[341] = 72'h3466ec522c273f310e; - mem[342] = 72'h368fd82225afcf35d0; - mem[343] = 72'h38b25f921f5e673a69; - mem[344] = 72'h3acea7f21931c73ed9; - mem[345] = 72'h3ce4d5721328a74324; - mem[346] = 72'h3ef50b020d41df474b; - mem[347] = 72'h40ff6a5a077c4f4b4f; - mem[348] = 72'h4304143201d6d74f31; - mem[349] = 72'h45032811fc507752f4; - mem[350] = 72'h46fcc4a1f6e82f5699; - mem[351] = 72'h48f10779f19d075a21; - mem[352] = 72'h4ae00d41ec6e1f5d8c; - mem[353] = 72'h4cc9f1d1e75a8f60dd; - mem[354] = 72'h4eaed011e261876414; - mem[355] = 72'h508ec221dd822f6732; - mem[356] = 72'h5269e151d8bbcf6a39; - mem[357] = 72'h54404641d40da76d29; - mem[358] = 72'h561208c1cf76f77002; - mem[359] = 72'h57df3ff1caf71772c7; - mem[360] = 72'h59a80259c68d5f7578; - mem[361] = 72'h5b6c65c9c239277815; - mem[362] = 72'h5d2c7f79bdf9df7a9f; - mem[363] = 72'h5ee86401b9cee77d18; - mem[364] = 72'h60a02771b5b7af7f7f; - mem[365] = 72'h6253dd49b1b3af81d5; - mem[366] = 72'h64039871adc25f841b; - mem[367] = 72'h65af6b61a9e33f8651; - mem[368] = 72'h67576811a615cf8878; - mem[369] = 72'h68fb9fe9a2599f8a91; - mem[370] = 72'h6a9c23f19eae2f8c9c; - mem[371] = 72'h6c3904b19b13178e99; - mem[372] = 72'h6dd252519787ef908a; - mem[373] = 72'h6f681c91940c3f926d; - mem[374] = 72'h70fa72a1909fb79445; - mem[375] = 72'h728963898d41df9610; - mem[376] = 72'h7414fdc989f26f97d1; - mem[377] = 72'h759d4f9186b0f79986; - mem[378] = 72'h772266c1837d2f9b30; - mem[379] = 72'h78a450c98056b79cd1; - mem[380] = 72'h7a231ae17d3d3f9e67; - mem[381] = 72'h7b9ed1d97a307f9ff4; - mem[382] = 72'h7d17824177301fa177; - mem[383] = 72'h7e8d3859743bdfa2f1; - mem[384] = 72'h00000003244857f8cf; - mem[385] = 72'h03242a83240d0fe8c0; - mem[386] = 72'h0647d94323530fd944; - mem[387] = 72'h096a900b221d2fc9ce; - mem[388] = 72'h0c8bd323206b9fba61; - mem[389] = 72'h0fab26eb1e3e9faafe; - mem[390] = 72'h12c810331b96879ba8; - mem[391] = 72'h15e214031873bf8c62; - mem[392] = 72'h18f8b7fb14d6c77d2e; - mem[393] = 72'h1c0b822b10c0276e0e; - mem[394] = 72'h1f19f93b0c30875f04; - mem[395] = 72'h2223a48b07288f5013; - mem[396] = 72'h25280c2301a917413d; - mem[397] = 72'h2826b8f2fbb2ef3285; - mem[398] = 72'h2b1f34b2f5470723ec; - mem[399] = 72'h2e110a2aee664f1576; - mem[400] = 72'h30fbc512e711e70723; - mem[401] = 72'h33def252df4ae6f8f7; - mem[402] = 72'h36ba1fdad71286eaf3; - mem[403] = 72'h398cdd02ce69fedd1a; - mem[404] = 72'h3c56ba3ac552b6cf6e; - mem[405] = 72'h3f174982bbce0ec1f1; - mem[406] = 72'h41ce1e32b1dd76b4a5; - mem[407] = 72'h447acd22a7827ea78c; - mem[408] = 72'h471cecb29cbec69aa9; - mem[409] = 72'h49b415029193e68dfc; - mem[410] = 72'h4c3fdfc286039e8188; - mem[411] = 72'h4ebfe8727a0fbe7550; - mem[412] = 72'h5133cc626dba0e6954; - mem[413] = 72'h539b2ac26104865d97; - mem[414] = 72'h55f5a4a253f116521a; - mem[415] = 72'h5842dd224681be46e0; - mem[416] = 72'h5a82797238b88e3bea; - mem[417] = 72'h5cb420b22a97ae3139; - mem[418] = 72'h5ed77c621c214626d0; - mem[419] = 72'h60ec38020d579e1cb0; - mem[420] = 72'h62f20189fe3ce612da; - mem[421] = 72'h64e88901eed3860950; - mem[422] = 72'h66cf8101df1dd60014; - mem[423] = 72'h68a69e61cf1e45f727; - mem[424] = 72'h6a6d9881bed74dee8a; - mem[425] = 72'h6c242941ae4b6de63f; - mem[426] = 72'h6dca0cf19d7d2dde47; - mem[427] = 72'h6f5f02918c6f2dd6a3; - mem[428] = 72'h70e2cba97b240dcf54; - mem[429] = 72'h72552c69699e75c85b; - mem[430] = 72'h73b5ebb157e11dc1bb; - mem[431] = 72'h7504d33145eeb5bb72; - mem[432] = 72'h7641af2933ca0db584; - mem[433] = 72'h776c4ec92175f5afef; - mem[434] = 72'h788484010ef535aab6; - mem[435] = 72'h798a23a0fc4aada5d9; - mem[436] = 72'h7a7d0548e9793da159; - mem[437] = 72'h7b5d0390d683cd9d37; - mem[438] = 72'h7c29fbe0c36d4d9973; - mem[439] = 72'h7ce3cea0b038a5960d; - mem[440] = 72'h7d8a5f389ce8cd9307; - mem[441] = 72'h7e1d93e08980cd9060; - mem[442] = 72'h7e9d55f07603958e1a; - mem[443] = 72'h7f0991c06274258c34; - mem[444] = 72'h7f6236884ed58d8aaf; - mem[445] = 72'h7fa736b03b2ac5898b; - mem[446] = 72'h7fd887882776e588c9; - mem[447] = 72'h7ff6218013bced8867; - mem[448] = 72'h07ff5588ffc03ffde1; - mem[449] = 72'h08ff0d40ffaf47fda1; - mem[450] = 72'h09feb300ff9c4ffd62; - mem[451] = 72'h0afe44c8ff875ffd22; - mem[452] = 72'h0bfdc0a0ff706ffce3; - mem[453] = 72'h0cfd2490ff577ffca3; - mem[454] = 72'h0dfc6e90ff3c97fc64; - mem[455] = 72'h0efb9cb0ff1fb7fc25; - mem[456] = 72'h07fd5680ff016ff78d; - mem[457] = 72'h08fc3600febdd7f692; - mem[458] = 72'h09face00fe725ff598; - mem[459] = 72'h0af916a8fe1f1ff4a0; - mem[460] = 72'h0bf70828fdc41ff3a8; - mem[461] = 72'h0cf49ac8fd615ff2b3; - mem[462] = 72'h0df1c6d8fcf6f7f1bf; - mem[463] = 72'h0eee84b0fc84e7f0cd; - mem[464] = 72'h07f56640fc0d67deca; - mem[465] = 72'h08f0ee78fb03afdb19; - mem[466] = 72'h09eb5e38f9dc67d77a; - mem[467] = 72'h0ae49838f89827d3ee; - mem[468] = 72'h0bdc7fc0f7378fd077; - mem[469] = 72'h0cd2f8d8f5bb37cd17; - mem[470] = 72'h0dc7e820f423dfc9ce; - mem[471] = 72'h0ebb32e0f2723fc69f; - mem[472] = 72'h07d65f48f0add78416; - mem[473] = 72'h08c51c78eccdf778a0; - mem[474] = 72'h09afcc00e8925f6e17; - mem[475] = 72'h0a9615e8e402776485; - mem[476] = 72'h0b77a9b8df25ff5bf3; - mem[477] = 72'h0c543ee0da04f75465; - mem[478] = 72'h0d2b94d0d4a7774ddb; - mem[479] = 72'h0dfd7340cf15af4856; - mem[480] = 72'h0764d478c95fce83fb; - mem[481] = 72'h08284348bd7b0e7982; - mem[482] = 72'h08dfa408b1430e7603; - mem[483] = 72'h098abf50a4ef9e78ba; - mem[484] = 72'h0a29927898b26e80b0; - mem[485] = 72'h0abc48a08cb58e8cf6; - mem[486] = 72'h0b433350811b669c9c; - mem[487] = 72'h0bbec2a075fefeaec1; - mem[488] = 72'h0617c2406b453d99fc; - mem[489] = 72'h0679767058143deeb1; - mem[490] = 72'h06c94c50478cb64190; - mem[491] = 72'h0709e5a8399e3e8de0; - mem[492] = 72'h073dc1082e1336d146; - mem[493] = 72'h07671e2824a3770b06; - mem[494] = 72'h0787f1c01d014f3b68; - mem[495] = 72'h07a1e40016e18f6344; - mem[496] = 72'h03db2e7011c4d72203; - mem[497] = 72'h03e982780aecbf7643; - mem[498] = 72'h03f24cc006ae6fab4b; - mem[499] = 72'h03f7ab280412afcc31; - mem[500] = 72'h03faf050027a5fe06a; - mem[501] = 72'h03fced6001817fecc8; - mem[502] = 72'h03fe229800ea17f453; - mem[503] = 72'h03fede58008e17f8e9; - mem[504] = 72'h01ffa86000525ff92a; - mem[505] = 72'h01ffdfc0001e57fd7c; - mem[506] = 72'h01fff420000b2fff13; - mem[507] = 72'h01fffba800041fffa9; - mem[508] = 72'h01fffe68000187ffe0; - mem[509] = 72'h01ffff68000097fff4; - mem[510] = 72'h01ffffc800003ffffc; - mem[511] = 72'h01fffff0000017fffe; - end + wire [DATAW-1:0] rd_data, rd_data_r; - reg [71:0] rd_data, rd_data_r; + VX_rom #( + .DATAW (DATAW), + .SIZE (SIZE), + .OUT_REG (1), + .INIT_DATA (ROM_DATA) + ) rom ( + .clk (clk), + .read (enable), + .addr (addr), + .rdata (rd_data) + ); - always @(posedge clk) begin - if (enable) begin - rd_data <= mem[addr]; - rd_data_r <= rd_data; - end - end + VX_pipe_register #( + .DATAW (DATAW), + .DEPTH (1) + ) rd_reg ( + .clk (clk), + .reset (1'b0), + .enable (enable), + .data_in (rd_data), + .data_out (rd_data_r) + ); assign a0 = 32'({rd_data_r[71:43], 2'b0}); assign a1 = 27'($signed({rd_data_r[42:19], 2'b0})); diff --git a/hw/rtl/mufu/VX_mufu_unit.sv b/hw/rtl/mufu/VX_mufu_unit.sv index 5e390fe1d4..394f9fd17d 100644 --- a/hw/rtl/mufu/VX_mufu_unit.sv +++ b/hw/rtl/mufu/VX_mufu_unit.sv @@ -1,127 +1,93 @@ `include "VX_define.vh" module VX_mufu_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1 ) ( - input wire clk, - input wire reset, + input wire clk, + input wire reset, - VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], - - VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH] + VX_execute_if.slave execute_if, + VX_result_if.master result_if ); `UNUSED_SPARAM (INSTANCE_ID) - localparam BLOCK_SIZE = `VX_CFG_NUM_MUFU_BLOCKS; - localparam NUM_LANES = `VX_CFG_NUM_MUFU_LANES; - localparam LATENCY = `VX_CFG_MUFU_LATENCY; - localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); - localparam HDR_WIDTH = $bits(mufu_header_t); - - VX_execute_if #( - .data_t (mufu_execute_t) - ) per_block_execute_if[BLOCK_SIZE](); - - VX_lane_dispatch #( - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (NUM_LANES), - .OUT_BUF (PARTIAL_BW ? 3 : 0) - ) lane_dispatch ( - .clk (clk), - .reset (reset), - .dispatch_if(dispatch_if), - .execute_if (per_block_execute_if) - ); + localparam NUM_PES = `VX_CFG_NUM_MUFU_LANES; + localparam LATENCY = `VX_MUFU_LATENCY; + localparam HDR_WIDTH = $bits(sfu_header_t); - VX_result_if #( - .data_t (mufu_result_t) - ) per_block_result_if[BLOCK_SIZE](); + `STATIC_ASSERT ((NUM_LANES % NUM_PES) == 0, ("invalid parameter: NUM_LANES=%0d, NUM_MUFU_LANES=%0d", NUM_LANES, NUM_PES)) - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - wire [NUM_LANES-1:0][31:0] req_data; - wire [NUM_LANES-1:0][31:0] rsp_data; - wire [NUM_LANES-1:0][31:0] pe_data_in; - wire [NUM_LANES-1:0][31:0] pe_data_out; - wire [2:0] pe_fn; - wire pe_enable; - mufu_header_t rsp_hdr; + `UNUSED_VAR (execute_if.data.op_type) + `UNUSED_VAR (execute_if.data.op_args) + `UNUSED_VAR (execute_if.data.rs2_data) + `UNUSED_VAR (execute_if.data.rs3_data) - `UNUSED_VAR (per_block_execute_if[block_idx].data.op_args) - `UNUSED_VAR (per_block_execute_if[block_idx].data.rs2_data) - `UNUSED_VAR (per_block_execute_if[block_idx].data.rs3_data) - `UNUSED_VAR (per_block_execute_if[block_idx].data.op_type) + wire [NUM_LANES-1:0][31:0] req_data; + wire [NUM_LANES-1:0][31:0] rsp_data; + wire [NUM_PES-1:0][31:0] pe_data_in; + wire [NUM_PES-1:0][31:0] pe_data_out; + wire [INST_MUFU_BITS-1:0] pe_fn; + wire pe_enable; + sfu_header_t rsp_hdr; - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_req_data - wire [`VX_CFG_XLEN-1:0] rs1 = per_block_execute_if[block_idx].data.rs1_data[i]; - if (`VX_CFG_XLEN > 32) begin : g_unbox - assign req_data[i] = (&rs1[`VX_CFG_XLEN-1:32]) ? rs1[31:0] : 32'h7fc00000; - end else begin : g_nobox - assign req_data[i] = rs1[31:0]; - end + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_req_data + wire [`VX_CFG_XLEN-1:0] rs1 = execute_if.data.rs1_data[i]; + if (`VX_CFG_XLEN > 32) begin : g_unbox + assign req_data[i] = (&rs1[`VX_CFG_XLEN-1:32]) ? rs1[31:0] : 32'h7fc00000; + end else begin : g_nobox + assign req_data[i] = rs1[31:0]; end + end - VX_pe_serializer #( - .NUM_LANES (NUM_LANES), - .NUM_PES (NUM_LANES), - .LATENCY (LATENCY), - .DATA_IN_WIDTH (32), - .DATA_OUT_WIDTH (32), - .SHARED_WIDTH (3), - .TAG_WIDTH (HDR_WIDTH), - .PE_REG (0), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) pe_serializer ( - .clk (clk), - .reset (reset), - .valid_in (per_block_execute_if[block_idx].valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .data_in (req_data), - .shared_in (per_block_execute_if[block_idx].data.op_type[2:0]), - .tag_in (per_block_execute_if[block_idx].data.header), - .ready_in (per_block_execute_if[block_idx].ready), - .pe_enable (pe_enable), - `UNUSED_PIN (pe_mask_out), - .pe_data_out(pe_data_in), - .pe_shared_out(pe_fn), - .pe_data_in (pe_data_out), - .valid_out (per_block_result_if[block_idx].valid), - `UNUSED_PIN (mask_out), - .data_out (rsp_data), - .tag_out (rsp_hdr), - .ready_out (per_block_result_if[block_idx].ready) - ); + VX_pe_serializer #( + .NUM_LANES (NUM_LANES), + .NUM_PES (NUM_PES), + .LATENCY (LATENCY), + .DATA_IN_WIDTH (32), + .DATA_OUT_WIDTH (32), + .SHARED_WIDTH (INST_MUFU_BITS), + .TAG_WIDTH (HDR_WIDTH), + .PE_REG (0), + .OUT_BUF (3) + ) pe_serializer ( + .clk (clk), + .reset (reset), + .valid_in (execute_if.valid), + .mask_in (execute_if.data.header.tmask), + .data_in (req_data), + .shared_in (execute_if.data.op_args.mufu.fn), + .tag_in (execute_if.data.header), + .ready_in (execute_if.ready), + .pe_enable (pe_enable), + `UNUSED_PIN (pe_mask_out), + .pe_data_out (pe_data_in), + .pe_shared_out (pe_fn), + .pe_data_in (pe_data_out), + .valid_out (result_if.valid), + `UNUSED_PIN (mask_out), + .data_out (rsp_data), + .tag_out (rsp_hdr), + .ready_out (result_if.ready) + ); - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_pes - VX_mufu_pe #( - .LATENCY (LATENCY) - ) pe ( - .clk (clk), - .reset (reset), - .enable (pe_enable), - .fn (pe_fn), - .dataa (pe_data_in[i]), - .result (pe_data_out[i]) - ); - end + for (genvar i = 0; i < NUM_PES; ++i) begin : g_pes + VX_mufu_pe pe ( + .clk (clk), + .reset (reset), + .enable (pe_enable), + .fn (pe_fn), + .dataa (pe_data_in[i]), + .result (pe_data_out[i]) + ); + end - assign per_block_result_if[block_idx].data.header = rsp_hdr; - for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data - if (`VX_CFG_XLEN > 32) begin : g_box - assign per_block_result_if[block_idx].data.data[i] = {{(`VX_CFG_XLEN-32){1'b1}}, rsp_data[i]}; - end else begin : g_nobox - assign per_block_result_if[block_idx].data.data[i] = rsp_data[i]; - end + assign result_if.data.header = rsp_hdr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data + if (`VX_CFG_XLEN > 32) begin : g_box + assign result_if.data.data[i] = {{(`VX_CFG_XLEN-32){1'b1}}, rsp_data[i]}; + end else begin : g_nobox + assign result_if.data.data[i] = rsp_data[i]; end end - VX_lane_gather #( - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (NUM_LANES), - .OUT_BUF (PARTIAL_BW ? 3 : 0) - ) lane_gather ( - .clk (clk), - .reset (reset), - .result_if (per_block_result_if), - .commit_if (commit_if) - ); - endmodule diff --git a/hw/scripts/gen_mufu_tables.py b/hw/scripts/gen_mufu_tables.py index 954c7c26cf..0011b85a11 100644 --- a/hw/scripts/gen_mufu_tables.py +++ b/hw/scripts/gen_mufu_tables.py @@ -25,7 +25,7 @@ Outputs (checked in, regenerate after any change here): sw/common/mufu_tables.h - hw/rtl/mufu/VX_mufu_rom.sv (no comments, by project convention for this unit) + hw/rtl/mufu/VX_mufu_rom.sv Usage: gen_mufu_tables.py write both outputs @@ -216,6 +216,10 @@ def emit_sv(entries, bases): iw = max(1, (max(n for _, _, n in bases) - 1).bit_length()) tw = max(1, (len(bases) - 1).bit_length()) w = W0 + W1 + W2 + words = [] + for i in range(ROM_DEPTH): + q0, q1, q2 = entries[i] if i < len(entries) else (0, 0, 0) + words.append((q0 << (W1 + W2)) | ((q1 & ((1 << W1) - 1)) << W2) | (q2 & ((1 << W2) - 1))) out = [] out.append("`include \"VX_platform.vh\"") out.append("") @@ -228,36 +232,51 @@ def emit_sv(entries, bases): out.append(f" output wire signed [{A1_BITS - 1}:0] a1,") out.append(f" output wire signed [{A2_BITS - 1}:0] a2") out.append(");") + out.append(f" localparam DATAW = {w};") + out.append(f" localparam SIZE = {ROM_DEPTH};") + out.append("") + out.append(" localparam [SIZE*DATAW-1:0] ROM_DATA = {") + for i in range(ROM_DEPTH - 1, -1, -1): + sep = "," if i > 0 else "" + out.append(f" {w}'h{words[i]:0{(w + 3) // 4}x}{sep}") + out.append(" };") out.append("") out.append(f" reg [{aw - 1}:0] base;") out.append("") out.append(" always @(*) begin") out.append(" case (tbl)") for i, (_, base, _) in enumerate(bases): - out.append(f" {tw}'d{i}: base = {aw}'d{base};") - out.append(f" default: base = '0;") + out.append(f" {tw}'d{i}: base = {aw}'d{base};") + out.append(f" default: base = '0;") out.append(" endcase") out.append(" end") out.append("") out.append(f" wire [{aw - 1}:0] addr = base + {aw}'(idx);") out.append("") - out.append(f" (* rom_style = \"block\" *) reg [{w - 1}:0] mem [0:{ROM_DEPTH - 1}];") - out.append("") - out.append(" initial begin") - for i in range(ROM_DEPTH): - q0, q1, q2 = entries[i] if i < len(entries) else (0, 0, 0) - v = (q0 << (W1 + W2)) | ((q1 & ((1 << W1) - 1)) << W2) | (q2 & ((1 << W2) - 1)) - out.append(f" mem[{i}] = {w}'h{v:0{(w + 3) // 4}x};") - out.append(" end") + out.append(" wire [DATAW-1:0] rd_data, rd_data_r;") out.append("") - out.append(f" reg [{w - 1}:0] rd_data, rd_data_r;") + out.append(" VX_rom #(") + out.append(" .DATAW (DATAW),") + out.append(" .SIZE (SIZE),") + out.append(" .OUT_REG (1),") + out.append(" .INIT_DATA (ROM_DATA)") + out.append(" ) rom (") + out.append(" .clk (clk),") + out.append(" .read (enable),") + out.append(" .addr (addr),") + out.append(" .rdata (rd_data)") + out.append(" );") out.append("") - out.append(" always @(posedge clk) begin") - out.append(" if (enable) begin") - out.append(" rd_data <= mem[addr];") - out.append(" rd_data_r <= rd_data;") - out.append(" end") - out.append(" end") + out.append(" VX_pipe_register #(") + out.append(" .DATAW (DATAW),") + out.append(" .DEPTH (1)") + out.append(" ) rd_reg (") + out.append(" .clk (clk),") + out.append(" .reset (1'b0),") + out.append(" .enable (enable),") + out.append(" .data_in (rd_data),") + out.append(" .data_out (rd_data_r)") + out.append(" );") out.append("") out.append(f" assign a0 = {A0_BITS}'({{rd_data_r[{w - 1}:{W1 + W2}], {S0}'b0}});") out.append(f" assign a1 = {A1_BITS}'($signed({{rd_data_r[{W1 + W2 - 1}:{W2}], {S1}'b0}}));") diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 9a2cde4690..ff5cb1b333 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -61,8 +61,6 @@ dxa_PKG = $(RTL_DIR)/dxa/VX_dxa_pkg.sv fpu_IP := 1 fpu_PRJ = $(if $(filter -DVX_CFG_FPU_TYPE_FPNEW,$(XCONFIGS)),VX_fpu_fpnew,$(if $(filter -DVX_CFG_FPU_TYPE_STD,$(XCONFIGS)),VX_fpu_std,VX_fpu_dsp)) fpu_INC = $(BASE_INC) $(FPU_INC) -# BASE_INC pulls in hw/rtl/VX_cluster.sv, which imports VX_tlb_pkg. -fpu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv mufu_PRJ := VX_mufu_unit_top mufu_CFG := -DVX_CFG_EXT_MUFU_ENABLE diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index d67a94c75b..0403566afe 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -9,7 +9,7 @@ # synthesizes, so both gates measure the same modules and a divergence between # them is meaningful rather than an artefact of two DUT lists. -DUTS := cache core tcu gfx tex raster om rtu dxa vm tensor top vortex +DUTS := cache core tcu gfx tex raster om rtu dxa mufu vm tensor top vortex UNITTEST_DIR ?= $(VORTEX_HOME)/hw/unittest AFU_RTL_DIR ?= $(VORTEX_HOME)/hw/rtl/afu @@ -60,6 +60,10 @@ dxa_TOP := VX_dxa_core_top dxa_INC := -I$(UNITTEST_DIR)/dxa_core dxa_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_DXA_ENABLE +mufu_TOP := VX_mufu_unit_top +mufu_INC := -I$(UNITTEST_DIR)/mufu_unit +mufu_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_MUFU_ENABLE + vm_TOP := VX_vm_top vm_INC := -I$(UNITTEST_DIR)/vm # Mirrors hw/syn/xilinx/dut/catalog.mk's vm_CFG. L2_ENABLE is load-bearing, not diff --git a/hw/unittest/fcvt_unit/main.cpp b/hw/unittest/fcvt_unit/main.cpp index d9b21d8f62..f81707deaa 100644 --- a/hw/unittest/fcvt_unit/main.cpp +++ b/hw/unittest/fcvt_unit/main.cpp @@ -112,8 +112,12 @@ int main(int argc, char** argv) { tests.push_back({a, f, op}); for (size_t i = 0; i < num_random; ++i) { uint32_t a = static_cast(rng()); - if (op == H2F && (i % 16) != 0) a |= 0xffff0000u; - if (op == F2H && (i % 2) == 0) a = (a & 0x807fffffu) | ((97u + (a >> 24) % 50u) << 23); + if (op == H2F && (i % 16) != 0) { + a |= 0xffff0000u; + } + if (op == F2H && (i % 2) == 0) { + a = (a & 0x807fffffu) | ((97u + (a >> 24) % 50u) << 23); + } tests.push_back({a, static_cast(i % 5), op}); } } @@ -138,8 +142,12 @@ int main(int argc, char** argv) { uint32_t ff = 0; uint32_t ga = t.a; #ifdef LEAN - if (t.op == F2I_S || t.op == F2I_U || t.op == F2H) ga = flush_sub(t.a); // DAZ float source - if (t.op == H2F && (t.a >> 16) == 0xffff) ga = 0xffff0000u | flush_sub16(uint16_t(t.a)); + if (t.op == F2I_S || t.op == F2I_U || t.op == F2H) { + ga = flush_sub(t.a); // DAZ float source + } + if (t.op == H2F && (t.a >> 16) == 0xffff) { + ga = 0xffff0000u | flush_sub16(uint16_t(t.a)); + } #endif uint32_t r = golden(t.op, ga, t.frm, &ff); #ifdef LEAN @@ -168,7 +176,9 @@ int main(int argc, char** argv) { } printf("fcvt_unit: checked=%zu errors=%zu per-op:", checked, errors); - for (int op = 0; op < (int)NUM_OPS; ++op) printf(" %s=%zu", op_name[op], per_op[op]); + for (int op = 0; op < (int)NUM_OPS; ++op) { + printf(" %s=%zu", op_name[op], per_op[op]); + } printf("\n%s\n", errors ? "FAILED" : "PASSED"); return errors ? 1 : 0; } diff --git a/hw/unittest/mufu_unit/VX_mufu_unit_top.sv b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv index 0d2e2c0567..0097f58929 100644 --- a/hw/unittest/mufu_unit/VX_mufu_unit_top.sv +++ b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv @@ -1,37 +1,41 @@ `include "VX_define.vh" module VX_mufu_unit_top import VX_gpu_pkg::*; ( - input wire clk, - input wire reset, + input wire clk, + input wire reset, - input wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_valid, - output wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_ready, - input wire [`VX_CFG_ISSUE_WIDTH-1:0][$bits(dispatch_t)-1:0] dispatch_data, + input wire execute_valid, + output wire execute_ready, + input wire [$bits(sfu_execute_t)-1:0] execute_data, - output wire [`VX_CFG_ISSUE_WIDTH-1:0] commit_valid, - input wire [`VX_CFG_ISSUE_WIDTH-1:0] commit_ready, - output wire [`VX_CFG_ISSUE_WIDTH-1:0][$bits(commit_t)-1:0] commit_data + output wire result_valid, + input wire result_ready, + output wire [$bits(sfu_result_t)-1:0] result_data ); - VX_dispatch_if dispatch_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if [`VX_CFG_ISSUE_WIDTH](); + VX_execute_if #( + .data_t (sfu_execute_t) + ) execute_if(); - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_ports - assign dispatch_if[i].valid = dispatch_valid[i]; - assign dispatch_if[i].data = dispatch_data[i]; - assign dispatch_ready[i] = dispatch_if[i].ready; + VX_result_if #( + .data_t (sfu_result_t) + ) result_if(); - assign commit_valid[i] = commit_if[i].valid; - assign commit_data[i] = commit_if[i].data; - assign commit_if[i].ready = commit_ready[i]; - end + assign execute_if.valid = execute_valid; + assign execute_if.data = execute_data; + assign execute_ready = execute_if.ready; + + assign result_valid = result_if.valid; + assign result_data = result_if.data; + assign result_if.ready = result_ready; VX_mufu_unit #( - .INSTANCE_ID ("mufu_unit_top") + .INSTANCE_ID ("mufu_unit_top"), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES) ) mufu_unit ( - .clk (clk), - .reset (reset), - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset), + .execute_if (execute_if), + .result_if (result_if) ); endmodule diff --git a/hw/unittest/mufu_unit/main.cpp b/hw/unittest/mufu_unit/main.cpp index b18596f520..d9045aa490 100644 --- a/hw/unittest/mufu_unit/main.cpp +++ b/hw/unittest/mufu_unit/main.cpp @@ -36,12 +36,12 @@ namespace { constexpr int kLatency = 7; const uint32_t kSpecials[] = { - 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, - 0x3f800000, 0xbf800000, 0x3f000000, 0x3fc00000, 0x40000000, 0x40800000, - 0x40490fdb, 0x3eaaaaab, 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, - 0x7fc00000, 0xffc00000, 0x7fa00000, 0x42fe0000, 0x43000000, 0xc2fc0000, - 0xc2fe0000, 0xc3000000, 0x4b000001, 0x4b800001, 0x3f7fffff, 0x3f800001, - 0x39800000, 0x39000000, 0x41000000, 0x40ffffff, 0x3a7fffff, 0x3a800000, + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, + 0x3f800000, 0xbf800000, 0x3f000000, 0x3fc00000, 0x40000000, 0x40800000, + 0x40490fdb, 0x3eaaaaab, 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, + 0x7fc00000, 0xffc00000, 0x7fa00000, 0x42fe0000, 0x43000000, 0xc2fc0000, + 0xc2fe0000, 0xc3000000, 0x4b000001, 0x4b800001, 0x3f7fffff, 0x3f800001, + 0x39800000, 0x39000000, 0x41000000, 0x40ffffff, 0x3a7fffff, 0x3a800000, }; struct Req { uint32_t fn, x; }; @@ -49,64 +49,75 @@ struct Req { uint32_t fn, x; }; } // namespace int main(int argc, char** argv) { - Verilated::commandArgs(argc, argv); - - uint64_t stride = 65521; - size_t num_random = 200000; - if (argc > 1) stride = std::strtoull(argv[1], nullptr, 0); - if (argc > 2) num_random = std::strtoull(argv[2], nullptr, 0); - if (stride == 0) stride = 1; - - std::vector reqs; - for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { - for (uint32_t x : kSpecials) - reqs.push_back({fn, x}); - for (uint64_t x = 0; x < (1ull << 32); x += stride) - reqs.push_back({fn, uint32_t(x)}); + Verilated::commandArgs(argc, argv); + + uint64_t stride = 65521; + size_t num_random = 200000; + if (argc > 1) { + stride = std::strtoull(argv[1], nullptr, 0); + } + if (argc > 2) { + num_random = std::strtoull(argv[2], nullptr, 0); + } + if (stride == 0) { + stride = 1; + } + + std::vector reqs; + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { + for (uint32_t x : kSpecials) { + reqs.push_back({fn, x}); } - std::mt19937 rng(0x6d756675u); - for (size_t i = 0; i < num_random; ++i) - reqs.push_back({uint32_t(i % FN_COUNT), uint32_t(rng())}); - - vl_simulator sim; - uint64_t ticks = 0; - sim->enable = 1; - sim->fn = 0; - sim->dataa = 0; - ticks = sim.reset(ticks); - - std::deque inflight; - size_t feed = 0, checked = 0; - size_t errors[FN_COUNT] = {}; - size_t total_errors = 0; - - for (size_t c = 0; c < reqs.size() + kLatency; ++c) { - if (feed < reqs.size()) { - sim->fn = reqs[feed].fn; - sim->dataa = reqs[feed].x; - inflight.push_back(reqs[feed]); - ++feed; - } - ticks = sim.step(ticks, 2); - - if (c >= size_t(kLatency - 1) && !inflight.empty()) { - Req r = inflight.front(); - inflight.pop_front(); - uint32_t got = sim->result; - uint32_t exp = eval(r.fn, r.x); - if (got != exp) { - if (total_errors < 20) - printf("MISMATCH %s x=%08x : got %08x expected %08x\n", fn_name(r.fn), r.x, got, exp); - ++errors[r.fn]; - ++total_errors; - } - ++checked; + for (uint64_t x = 0; x < (1ull << 32); x += stride) { + reqs.push_back({fn, uint32_t(x)}); + } + } + std::mt19937 rng(0x6d756675u); + for (size_t i = 0; i < num_random; ++i) { + reqs.push_back({uint32_t(i % FN_COUNT), uint32_t(rng())}); + } + + vl_simulator sim; + uint64_t ticks = 0; + sim->enable = 1; + sim->fn = 0; + sim->dataa = 0; + ticks = sim.reset(ticks); + + std::deque inflight; + size_t feed = 0, checked = 0; + size_t errors[FN_COUNT] = {}; + size_t total_errors = 0; + + for (size_t c = 0; c < reqs.size() + kLatency; ++c) { + if (feed < reqs.size()) { + sim->fn = reqs[feed].fn; + sim->dataa = reqs[feed].x; + inflight.push_back(reqs[feed]); + ++feed; + } + ticks = sim.step(ticks, 2); + + if (c >= size_t(kLatency - 1) && !inflight.empty()) { + Req r = inflight.front(); + inflight.pop_front(); + uint32_t got = sim->result; + uint32_t exp = eval(r.fn, r.x); + if (got != exp) { + if (total_errors < 20) { + printf("MISMATCH %s x=%08x : got %08x expected %08x\n", fn_name(r.fn), r.x, got, exp); } + ++errors[r.fn]; + ++total_errors; + } + ++checked; } - - printf("mufu_unit: checked=%zu errors=%zu [", checked, total_errors); - for (uint32_t fn = 0; fn < FN_COUNT; ++fn) - printf("%s%s=%zu", fn ? " " : "", fn_name(fn), errors[fn]); - printf("]\n%s\n", total_errors ? "FAILED" : "PASSED"); - return total_errors ? 1 : 0; + } + + printf("mufu_unit: checked=%zu errors=%zu [", checked, total_errors); + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { + printf("%s%s=%zu", fn ? " " : "", fn_name(fn), errors[fn]); + } + printf("]\n%s\n", total_errors ? "FAILED" : "PASSED"); + return total_errors ? 1 : 0; } diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index b23c4c4dd8..ec729d83e3 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -47,9 +47,6 @@ #ifdef VX_CFG_EXT_TCU_ENABLE #include "tcu_unit.h" #endif -#ifdef VX_CFG_EXT_MUFU_ENABLE -#include "mufu_unit.h" -#endif #ifdef VX_CFG_VM_ENABLE #include "mem/mmu.h" #include @@ -277,11 +274,6 @@ class Core::Impl { #endif #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - snprintf(sname, 100, "%s-mufu", name.c_str()); - func_units_.at((int)FUType::MUFU) = SimPlatform::instance().create_object(sname, simobject_); - #endif - // initialize dispatchers after the units they feed, so a unit takes its // input before the dispatcher fills it in the same cycle. { @@ -294,10 +286,6 @@ class Core::Impl { dispatchers_.at((int)FUType::SFU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_SFU_BLOCKS, VX_CFG_NUM_SFU_LANES, 1); #ifdef VX_CFG_EXT_TCU_ENABLE dispatchers_.at((int)FUType::TCU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_TCU_BLOCKS, VX_CFG_NUM_TCU_LANES, 1); - #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - uint32_t mufu_dly = MufuUnit::kGather ? 2 : 1; - dispatchers_.at((int)FUType::MUFU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_MUFU_BLOCKS, VX_CFG_NUM_MUFU_LANES, mufu_dly); #endif for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { auto& dispatch = dispatchers_.at(fu); @@ -582,9 +570,6 @@ class Core::Impl { case FUType::SFU: ++perf_stats_.sfu_stalls; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: ++perf_stats_.tcu_stalls; break; - #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - case FUType::MUFU: ++perf_stats_.mufu_stalls; break; #endif default: assert(false); } @@ -820,15 +805,12 @@ class Core::Impl { // process completed instructions: one beat per issue slice per cycle, // granted fixed-priority in EX-unit order (ALU highest, then LSU, SFU, - // FPU, TCU, MUFU) — a busy higher class starves lower ones' writebacks. + // FPU, TCU) — a busy higher class starves lower ones' writebacks. static constexpr FUType kCommitPrio[] = { FUType::ALU, FUType::LSU, FUType::SFU, FUType::FPU, #ifdef VX_CFG_EXT_TCU_ENABLE FUType::TCU, #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - FUType::MUFU, - #endif }; for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { SimChannel* granted = nullptr; @@ -886,9 +868,6 @@ class Core::Impl { case FUType::SFU: ++perf_stats_.sfu_instrs; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: ++perf_stats_.tcu_instrs; break; - #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - case FUType::MUFU: ++perf_stats_.mufu_instrs; break; #endif default: assert(false); } diff --git a/sim/simx/core.h b/sim/simx/core.h index 48936b8fae..df1e117b7b 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -55,9 +55,6 @@ class Core : public SimObject { uint64_t sfu_stalls = 0; #ifdef VX_CFG_EXT_TCU_ENABLE uint64_t tcu_stalls = 0; - #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - uint64_t mufu_stalls = 0; #endif uint64_t branches = 0; uint64_t divergence = 0; @@ -67,9 +64,6 @@ class Core : public SimObject { uint64_t sfu_instrs = 0; #ifdef VX_CFG_EXT_TCU_ENABLE uint64_t tcu_instrs = 0; - #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - uint64_t mufu_instrs = 0; #endif uint64_t ifetches = 0; uint64_t loads = 0; diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 74d9b8ea72..c000ef2fa5 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -248,14 +248,6 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { } } break; #endif - #ifdef VX_CFG_EXT_MUFU_ENABLE - case VX_DCR_MPM_CLASS_MUFU: { - switch (addr) { - CSR_READ_64(VX_CSR_MPM_MUFU_STALLS, core_perf.mufu_stalls); - CSR_READ_64(VX_CSR_MPM_MUFU_INSTRS, core_perf.mufu_instrs); - } - } break; - #endif #ifdef VX_CFG_EXT_DXA_ENABLE case VX_DCR_MPM_CLASS_DXA: { auto cluster_perf = core_->socket()->cluster()->perf_stats(); @@ -406,42 +398,47 @@ void CsrUnit::process(instr_trace_t* trace) { uint32_t num_threads = VX_CFG_NUM_THREADS; auto& rs1_data = trace->src_data[0]; - uint32_t thread_start = 0; - for (; thread_start < num_threads; ++thread_start) { - if (tmask.test(thread_start)) break; - } - trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; - if (thread_start == num_threads) - return; - - // CSRs are per warp: every active lane reads the old value, and the write - // takes the first active lane's operand, as in the RTL. - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - rd_data[t].i = this->get_csr(csr_addr, wid, t); - } - Word csr_value = rd_data[thread_start].i; - Word src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[thread_start].i; - - switch (csr_type) { - case CsrType::CSRRW: - this->set_csr(csr_addr, src_v, wid, thread_start); - break; - case CsrType::CSRRS: - if (src_v != 0) { - this->set_csr(csr_addr, csr_value | src_v, wid, thread_start); + // The warp is processed one SFU-width packet at a time: a packet's active + // lanes read the CSR, then its first active lane supplies the write. + constexpr uint32_t kPacketLanes = VX_CFG_NUM_SFU_LANES; + for (uint32_t base = 0; base < num_threads; base += kPacketLanes) { + uint32_t first = num_threads; + for (uint32_t t = base; t < base + kPacketLanes; ++t) { + if (tmask.test(t)) { + first = t; + break; + } } - break; - case CsrType::CSRRC: - if (src_v != 0) { - this->set_csr(csr_addr, csr_value & ~src_v, wid, thread_start); + if (first == num_threads) { + continue; + } + for (uint32_t t = first; t < base + kPacketLanes; ++t) { + if (tmask.test(t)) { + rd_data[t].i = this->get_csr(csr_addr, wid, t); + } + } + Word csr_value = rd_data[first].i; + Word src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[first].i; + switch (csr_type) { + case CsrType::CSRRW: + this->set_csr(csr_addr, src_v, wid, first); + break; + case CsrType::CSRRS: + if (src_v != 0) { + this->set_csr(csr_addr, csr_value | src_v, wid, first); + } + break; + case CsrType::CSRRC: + if (src_v != 0) { + this->set_csr(csr_addr, csr_value & ~src_v, wid, first); + } + break; + default: + std::abort(); } - break; - default: - std::abort(); } DT(3, "csr-unit process: op=" << csr_type << ", " << *trace); } diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 7fe704d961..630ae1bbd9 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -220,8 +220,12 @@ static op_string_t op_string(const Instr &instr) { } } case FpuType::F2F: { - if (fpuArgs.is_f16) return {(fpuArgs.cvt == 1) ? "FCVT.H.D":"FCVT.H.S", ""}; - if (fpuArgs.cvt == 2) return {fpuArgs.is_f64 ? "FCVT.D.H":"FCVT.S.H", ""}; + if (fpuArgs.is_f16) { + return {(fpuArgs.cvt == 1) ? "FCVT.H.D":"FCVT.H.S", ""}; + } + if (fpuArgs.cvt == 2) { + return {fpuArgs.is_f64 ? "FCVT.D.H":"FCVT.S.H", ""}; + } return {fpuArgs.is_f64 ? "FCVT.D.S":"FCVT.S.D", ""}; } case FpuType::FCMP: { @@ -262,8 +266,12 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); - if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) return {"PACKLB.F", ""}; - if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) return {"PACKLH.F", ""}; + if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { + return {"PACKLB.F", ""}; + } + if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) { + return {"PACKLH.F", ""}; + } if (dst_float) { switch (lsuArgs.width) { case 1: return {"FLH", to_hex_str(lsuArgs.offset)}; @@ -412,7 +420,7 @@ static op_string_t op_string(const Instr &instr) { #ifdef VX_CFG_EXT_MUFU_ENABLE ,[&](MufuType mufu_type)-> op_string_t { std::ostringstream os; - os << mufu_type << ".S"; + os << mufu_type; return {os.str(), ""}; } #endif @@ -768,25 +776,27 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { break; case 0x20: // FCVT.S.D, FCVT.S.H case 0x21: // FCVT.D.S, FCVT.D.H + case 0x22: { // FCVT.H.S, FCVT.H.D + // fmt codes: 0 = S, 1 = D, 2 = H + uint32_t dst_fmt = funct7 & 0x3; + uint32_t src_fmt = rs2; + bool supported = (src_fmt <= 2) && (src_fmt != dst_fmt); #ifndef VX_CFG_EXT_ZFHMIN_ENABLE - if ((rs2 & 0x3) == 0x2) - std::abort(); + supported = supported && (dst_fmt != 2) && (src_fmt != 2); #endif + if (!supported) { + std::abort(); + } instr->set_op_type(FpuType::F2F); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); - break; + } break; #ifdef VX_CFG_EXT_ZFHMIN_ENABLE - case 0x22: // FCVT.H.S, FCVT.H.D - instr->set_op_type(FpuType::F2F); - instr->set_dest_reg(rd, RegType::Float); - instr->set_src_reg(0, rs1, RegType::Float); - instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); - break; case 0x72: // FMV.X.H - if (funct3 != 0) + if (funct3 != 0) { std::abort(); + } instr->set_op_type(FpuType::FMVXW); instr->set_dest_reg(rd, RegType::Integer); instr->set_src_reg(0, rs1, RegType::Float); @@ -1182,11 +1192,11 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } break; #ifdef VX_CFG_EXT_MUFU_ENABLE case Opcode::EXT3: { - if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) + if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) { std::abort(); - instr->set_fu_type(FUType::MUFU); + } + instr->set_fu_type(FUType::SFU); instr->set_op_type(MufuType(rs2)); - instr->set_args(IntrMufuArgs{funct3}); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); } break; diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 0e29004275..6316092c57 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "debug.h" #include "core.h" #include "csr_unit.h" @@ -27,26 +28,17 @@ using namespace vortex; namespace { inline uint64_t nan_box(uint32_t value) { - return value | 0xffffffff00000000; -} -inline bool is_nan_boxed(uint64_t value) { - return (uint32_t(value >> 32) == 0xffffffff); + return nanbox::box32(value); } inline int64_t check_boxing(int64_t a) { - if (is_nan_boxed(a)) - return a; - return nan_box(0x7fc00000); // NaN + return nanbox::box32(nanbox::unbox32(a, 0x7fc00000)); } -#ifdef VX_CFG_EXT_ZFHMIN_ENABLE inline uint64_t nan_box16(uint16_t value) { - return value | 0xffffffffffff0000; + return nanbox::box16(value); } inline uint16_t check_boxing16(uint64_t a) { - if ((a >> 16) == 0xffffffffffff) - return uint16_t(a); - return 0x7e00; + return nanbox::unbox16(a, 0x7e00); } -#endif } FpuUnit::FpuUnit(const SimContext& ctx, const char* name, Core* core) @@ -281,7 +273,6 @@ void FpuUnit::execute(instr_trace_t* trace) { if (!tmask.test(t)) continue; uint32_t frm = emu.get_fpu_rm(fpuArgs.frm, wid, t); uint32_t fflags = 0; -#ifdef VX_CFG_EXT_ZFHMIN_ENABLE if (fpuArgs.is_f16) { if (fpuArgs.cvt == 1) { rd_data[t].u64 = nan_box16(rv_dtoh(rs1_data[t].u64, frm, &fflags)); @@ -295,9 +286,7 @@ void FpuUnit::execute(instr_trace_t* trace) { } else { rd_data[t].u64 = nan_box(rv_htof_s(h, frm, &fflags)); } - } else -#endif - if (fpuArgs.is_f64) { + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rv_ftod(check_boxing(rs1_data[t].u64), frm, &fflags); } else { rd_data[t].u64 = nan_box(rv_dtof(rs1_data[t].u64, frm, &fflags)); @@ -321,12 +310,9 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; -#ifdef VX_CFG_EXT_ZFHMIN_ENABLE if (fpuArgs.is_f16) { rd_data[t].i = sext((uint64_t)uint16_t(rs1_data[t].u64), 16); - } else -#endif - if (fpuArgs.is_f64) { + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].u64; } else { uint32_t result = (uint32_t)rs1_data[t].u64; @@ -339,12 +325,9 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; -#ifdef VX_CFG_EXT_ZFHMIN_ENABLE if (fpuArgs.is_f16) { rd_data[t].u64 = nan_box16(uint16_t(rs1_data[t].i)); - } else -#endif - if (fpuArgs.is_f64) { + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].i; } else { rd_data[t].u64 = nan_box((uint32_t)rs1_data[t].i); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 88e8f63207..c891fc3f51 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "debug.h" #include "core.h" #include "local_mem_switch.h" @@ -253,6 +254,7 @@ void LsuUnit::process_response_step(uint32_t b) { uint32_t byte_off = (data_bs == 0 || data_bs == 0x0F) ? 0 : (uint32_t)__builtin_ctz(data_bs); bool nan_box = dst_float && (data_bytes < 8); + bool half_float = VX_CFG_EXT_ZFHMIN_ENABLED && dst_float && (entry.lsu_args.width == 1); for (uint32_t lane = 0; lane < lsu_rsp.mask.size(); ++lane) { if (!lsu_rsp.mask.test(lane)) @@ -268,8 +270,8 @@ void LsuUnit::process_response_step(uint32_t b) { case 0: // LB formatted = (uint64_t)(int64_t)sext((Word)read_data, data_width); break; - case 1: // LH (FLH: raw bits for Float dest, NaN-boxed below) - formatted = dst_float ? read_data + case 1: // LH (raw bits for a Zfhmin Float dest, NaN-boxed below) + formatted = half_float ? read_data : (uint64_t)(int64_t)sext((Word)read_data, data_width); break; case 2: // LW (sign-ext for Integer dest; raw bits for Float dest, NaN-boxed below) @@ -287,10 +289,9 @@ void LsuUnit::process_response_step(uint32_t b) { } // Place at bytesel-specified position; OR-in NaN-box for Float dest. auto& dst = entry.trace->dst_data.at(lane_info.tid); - uint64_t box = !nan_box ? 0 - : (entry.lsu_args.width == 1) ? 0xFFFFFFFFFFFF0000ull - : 0xFFFFFFFF00000000ull; - dst.u64 = (formatted << (8 * byte_off)) | box; + uint32_t box_bits = half_float ? 16 : 32; + dst.u64 = (formatted << (8 * byte_off)) + | (nan_box ? nanbox::box(0, box_bits) : 0); } } entry.count -= lsu_rsp.mask.count(); // track remaining diff --git a/sim/simx/mufu_unit.cpp b/sim/simx/mufu_unit.cpp index 7ce471867f..79a3f52ef3 100644 --- a/sim/simx/mufu_unit.cpp +++ b/sim/simx/mufu_unit.cpp @@ -13,62 +13,49 @@ #include "mufu_unit.h" #include +#include #include "debug.h" -#include "core.h" using namespace vortex; -namespace { -// SimX keeps every single-precision FP register value NaN-boxed (see fpu_unit.cpp). -inline uint32_t unbox(uint64_t value) { - if (uint32_t(value >> 32) != 0xffffffff) - return mufu::QNAN_F32; - return uint32_t(value); -} -inline uint64_t box(uint32_t value) { - return value | 0xffffffff00000000; -} +uint32_t MufuUnit::issue_cycles(const instr_trace_t* trace) { + constexpr uint32_t kPacketLanes = VX_CFG_NUM_SFU_LANES; + constexpr uint32_t kBatches = VX_CFG_NUM_SFU_LANES / VX_CFG_NUM_MUFU_LANES; + static_assert(VX_CFG_NUM_SFU_LANES % VX_CFG_NUM_MUFU_LANES == 0, + "MUFU lanes must divide the SFU lanes"); + uint32_t cycles = 0; + for (uint32_t base = 0; base < VX_CFG_NUM_THREADS; base += kPacketLanes) { + for (uint32_t t = base; t < base + kPacketLanes; ++t) { + if (trace->tmask.test(t)) { + cycles += kBatches; + break; + } + } + } + return cycles; } -MufuUnit::MufuUnit(const SimContext& ctx, const char* name, Core* core) - // The RTL pipeline has no tag store: what is in flight is bounded by the - // pipeline depth plus its two-entry output buffer. - : FuncUnit(ctx, name, core, VX_CFG_MUFU_LATENCY + 2) -{} +bool MufuUnit::process(instr_trace_t* trace) { + if (head_ != trace) { + head_ = trace; + remaining_ = issue_cycles(trace); + } + if (remaining_ > 1) { + --remaining_; + return false; + } + head_ = nullptr; + remaining_ = 0; -void MufuUnit::execute(instr_trace_t* trace) { - auto& tmask = trace->tmask; - auto mufu_type = std::get(trace->op_type); + auto fn = uint32_t(std::get(trace->op_type)); auto& rs1_data = trace->src_data[0]; trace->dst_data.assign(VX_CFG_NUM_THREADS, reg_data_t{}); - auto& rd_data = trace->dst_data; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { - if (!tmask.test(t)) - continue; - rd_data[t].u64 = box(mufu::eval(uint32_t(mufu_type), unbox(rs1_data[t].u64))); - } - DT(3, this->name() << " execute: op=" << mufu_type << ", " << *trace); -} - -void MufuUnit::on_tick() { - bool idle = true; - for (uint32_t b = 0; b < VX_CFG_NUM_MUFU_BLOCKS; ++b) { - auto& input = Inputs.at(b); - if (!input.empty()) { - auto& output = Outputs.at(b); - if (!output.full()) { - auto trace = input.peek(); - this->execute(trace); - // Without a lane-gather stage commit reads this port directly, so the - // result also takes the registered stage other units cross into commit. - uint32_t delay = VX_CFG_MUFU_LATENCY + (kGather ? 0 : 1); - output.send(trace, delay); - input.pop(); - } + if (trace->tmask.test(t)) { + uint32_t x = nanbox::unbox32(rs1_data[t].u64, mufu::QNAN_F32); + trace->dst_data[t].u64 = nanbox::box32(mufu::eval(fn, x)); } - idle &= (input.size() == 0); - } - if (idle) { - this->tick_sleep(); } + DT(3, "mufu-unit process: " << *trace); + return true; } diff --git a/sim/simx/mufu_unit.h b/sim/simx/mufu_unit.h index 442daa4629..510f122669 100644 --- a/sim/simx/mufu_unit.h +++ b/sim/simx/mufu_unit.h @@ -13,24 +13,30 @@ #pragma once -#include "func_unit.h" +#include +#include +#include "types.h" +#include "instr_trace.h" namespace vortex { -class MufuUnit : public FuncUnit { +// MUFU sub-unit of the SFU. The SFU's lanes are serialized onto the MUFU +// datapath lanes, one batch per cycle, and a warp is accepted on its last batch. +class MufuUnit { public: - MufuUnit(const SimContext& ctx, const char* name, Core*); + // Cycles a result spends in the datapath and its registered output after the + // warp is accepted. + static constexpr uint32_t kLatency = VX_MUFU_LATENCY + 1; - // Partial bandwidth: a lane-gather stage takes results off the response - // port. Otherwise the commit arbiter reads the response port directly. - static constexpr bool kGather = (VX_CFG_NUM_MUFU_BLOCKS != VX_CFG_ISSUE_WIDTH) - || (VX_CFG_NUM_MUFU_LANES != VX_CFG_SIMD_WIDTH); - -protected: - void on_tick() override; + // Returns false while the warp's batches are still being issued; on the + // accepting cycle computes the results and returns true. + bool process(instr_trace_t* trace); private: - void execute(instr_trace_t* trace); + static uint32_t issue_cycles(const instr_trace_t* trace); + + const instr_trace_t* head_ = nullptr; + uint32_t remaining_ = 0; }; } diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index de61756629..2b680b1132 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -390,6 +390,21 @@ void SfuUnit::on_tick() { // The fragment work distributor (above + scheduler) launches fragment // warps directly from the autonomously-pulled covered-quad waves. +#ifdef VX_CFG_EXT_MUFU_ENABLE + if (std::get_if(&trace->op_type)) { + if (output.full()) { + continue; + } + if (!mufu_unit_.process(trace)) { + continue; + } + output.send(trace, this->latency_of(trace) + MufuUnit::kLatency); + trace->resume_warp = trace->fetch_stall; + input.pop(); + continue; + } +#endif + if (output.full()) continue; // stall — no side effects this tick diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 1eb74a31f1..9485f6bc19 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -35,6 +35,9 @@ #ifdef VX_CFG_EXT_RTU_ENABLE #include "rtu/rtu_unit.h" #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +#include "mufu_unit.h" +#endif namespace vortex { @@ -159,6 +162,9 @@ class SfuUnit : public FuncUnit { #ifdef VX_CFG_EXT_RTU_ENABLE std::unique_ptr rtu_unit_; #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE + MufuUnit mufu_unit_; +#endif }; } diff --git a/sim/simx/types.h b/sim/simx/types.h index 63b94bf42c..31f7fbddbf 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -191,9 +191,6 @@ enum class FUType { SFU, #ifdef VX_CFG_EXT_TCU_ENABLE TCU, -#endif -#ifdef VX_CFG_EXT_MUFU_ENABLE - MUFU, #endif Count }; @@ -206,9 +203,6 @@ inline std::ostream &operator<<(std::ostream &os, const FUType& type) { case FUType::SFU: os << "SFU"; break; #ifdef VX_CFG_EXT_TCU_ENABLE case FUType::TCU: os << "TCU"; break; -#endif -#ifdef VX_CFG_EXT_MUFU_ENABLE - case FUType::MUFU: os << "MUFU"; break; #endif default: assert(false); @@ -611,10 +605,6 @@ enum class MufuType { TANH = 6, }; -struct IntrMufuArgs { - uint32_t fmt : 3; -}; - inline std::ostream &operator<<(std::ostream &os, const MufuType& type) { switch (type) { case MufuType::RCP: os << "MUFU.RCP"; break; @@ -892,9 +882,6 @@ using IntrArgs = std::variant< #ifdef VX_CFG_EXT_TCU_ENABLE , IntrTcuArgs #endif -#ifdef VX_CFG_EXT_MUFU_ENABLE -, IntrMufuArgs -#endif #ifdef VX_CFG_EXT_TEX_ENABLE , IntrTexArgs #endif diff --git a/sw/common/mufu.cpp b/sw/common/mufu.cpp index ad5a906620..2896183dda 100644 --- a/sw/common/mufu.cpp +++ b/sw/common/mufu.cpp @@ -25,7 +25,7 @@ constexpr uint32_t POS_INF = 0x7f800000; constexpr uint32_t NEG_INF = 0xff800000; constexpr uint32_t ONE = 0x3f800000; -// Output of the range-reduction stage; mirrors the RTL stage-1 register. +// Output of the range-reduction stage. struct reduced_t { bool special; // result is `value`, bypassing interpolation uint32_t value; @@ -50,13 +50,18 @@ reduced_t reduce(uint32_t fn, uint32_t x) { auto special = [&](uint32_t v) { r.special = true; r.value = v; return r; }; - if (is_nan) + if (is_nan) { return special(QNAN_F32); + } switch (fn) { case FN_RCP: - if (is_inf) return special(s << 31); - if (is_zero) return special(s ? NEG_INF : POS_INF); + if (is_inf) { + return special(s << 31); + } + if (is_zero) { + return special(s ? NEG_INF : POS_INF); + } r.addr = TBL_RCP_BASE + (m >> 16); r.xl = (m & 0xffff) << 2; r.exp = 127 - int32_t(e); @@ -64,9 +69,15 @@ reduced_t reduce(uint32_t fn, uint32_t x) { return r; case FN_RSQRT: { - if (is_zero) return special(s ? NEG_INF : POS_INF); - if (s) return special(QNAN_F32); - if (is_inf) return special(0); + if (is_zero) { + return special(s ? NEG_INF : POS_INF); + } + if (s) { + return special(QNAN_F32); + } + if (is_inf) { + return special(0); + } int32_t k = int32_t(e) - 127; r.addr = TBL_RSQRT_BASE + ((uint32_t(k) & 1) << 6) + (m >> 17); r.xl = (m & 0x1ffff) << 1; @@ -75,9 +86,15 @@ reduced_t reduce(uint32_t fn, uint32_t x) { } case FN_EX2: { - if (is_inf) return special(s ? 0 : POS_INF); - if (is_zero) return special(ONE); - if (e >= 134) return special(s ? 0 : POS_INF); + if (is_inf) { + return special(s ? 0 : POS_INF); + } + if (is_zero) { + return special(ONE); + } + if (e >= 134) { + return special(s ? 0 : POS_INF); + } int32_t sh = int32_t(e) - 120; int64_t mag = (sh >= 0) ? (int64_t(sig) << sh) : ((sh > -24) ? int64_t(sig >> -sh) : 0); int64_t X = s ? -mag : mag; @@ -89,9 +106,15 @@ reduced_t reduce(uint32_t fn, uint32_t x) { } case FN_LG2: - if (is_zero) return special(NEG_INF); - if (s) return special(QNAN_F32); - if (is_inf) return special(POS_INF); + if (is_zero) { + return special(NEG_INF); + } + if (s) { + return special(QNAN_F32); + } + if (is_inf) { + return special(POS_INF); + } r.addr = TBL_LG2_BASE + (m >> 17); r.xl = (m & 0x1ffff) << 1; r.is_lg2 = true; @@ -101,19 +124,25 @@ reduced_t reduce(uint32_t fn, uint32_t x) { case FN_SINPI: case FN_COSPI: { bool is_sin = (fn == FN_SINPI); - if (is_inf) return special(QNAN_F32); - if (is_zero) return special(is_sin ? (s << 31) : ONE); + if (is_inf) { + return special(QNAN_F32); + } + if (is_zero) { + return special(is_sin ? (s << 31) : ONE); + } // u = 2|x| in quarter turns, 30 fraction bits, modulo 4 int32_t sh = int32_t(e) - 119; uint32_t u; - if (sh >= 32) + if (sh >= 32) { u = 0; - else if (sh >= 0) + } else if (sh >= 0) { u = uint32_t(uint64_t(sig) << sh); - else + } else { u = (sh > -24) ? (sig >> -sh) : 0; - if (!is_sin) + } + if (!is_sin) { u += (1u << FRAC); + } uint32_t q = u >> FRAC; uint32_t t = u & ((1u << FRAC) - 1); r.neg = ((q >> 1) & 1) ^ (is_sin ? s : 0); @@ -121,8 +150,9 @@ reduced_t reduce(uint32_t fn, uint32_t x) { // Odd quadrants need cos(pi/2 t) = sin(pi/2 (1 - t)); ~t stands in for // 1 - t, and t == 0 (exactly 1) is returned directly to stay exact. if (q & 1) { - if (t == 0) + if (t == 0) { return special(r.neg ? (ONE | 0x80000000) : ONE); + } t = ~t & ((1u << FRAC) - 1); } r.addr = TBL_SIN_BASE + (t >> 24); @@ -131,11 +161,19 @@ reduced_t reduce(uint32_t fn, uint32_t x) { } case FN_TANH: - if (is_inf) return special(s ? (ONE | 0x80000000) : ONE); - if (is_zero) return special(s << 31); + if (is_inf) { + return special(s ? (ONE | 0x80000000) : ONE); + } + if (is_zero) { + return special(s << 31); + } // |x| >= 8 rounds to +-1; below 2^-5, tanh x = x within 2^-11. - if (e >= 130) return special(s ? (ONE | 0x80000000) : ONE); - if (e < 122) return special(x); + if (e >= 130) { + return special(s ? (ONE | 0x80000000) : ONE); + } + if (e < 122) { + return special(x); + } // Entries hold 2^-(j+4) * tanh(2^j (1+t)) for binade j = e - 127. r.addr = TBL_TANH_BASE + ((e - 122) << 3) + (m >> 20); r.xl = (m >> 2) & ((1u << XL_BITS) - 1); @@ -161,8 +199,9 @@ int64_t interpolate(uint32_t addr, uint32_t xl) { } uint32_t normalize(uint64_t mag, int32_t exp, bool neg, bool zsign) { - if (mag == 0) + if (mag == 0) { return uint32_t(zsign) << 31; + } int lead = 63 - __builtin_clzll(mag); uint64_t norm = (mag << (39 - lead)) & ((uint64_t(1) << 40) - 1); uint32_t sig = uint32_t(norm >> 16); @@ -174,10 +213,12 @@ uint32_t normalize(uint64_t mag, int32_t exp, bool neg, bool zsign) { bexp += 1; } uint32_t sgn = uint32_t(neg) << 31; - if (bexp >= 255) + if (bexp >= 255) { return sgn | POS_INF; - if (bexp <= 0) + } + if (bexp <= 0) { return sgn; + } return sgn | (uint32_t(bexp) << 23) | (sig & 0x7fffff); } @@ -185,8 +226,9 @@ uint32_t normalize(uint64_t mag, int32_t exp, bool neg, bool zsign) { uint32_t eval(uint32_t fn, uint32_t x) { reduced_t r = reduce(fn, x); - if (r.special) + if (r.special) { return r.value; + } int64_t p = interpolate(r.addr, r.xl); if (r.is_lg2) { int64_t l = (int64_t(r.k) << FRAC) + p; diff --git a/sw/common/mufu.h b/sw/common/mufu.h index d642b66289..f633d07cf3 100644 --- a/sw/common/mufu.h +++ b/sw/common/mufu.h @@ -15,7 +15,7 @@ #include -// Bit-exact model of the MUFU datapath (hw/rtl/mufu). +// Bit-exact reference for the MUFU special functions. namespace vortex::mufu { enum fn_t : uint32_t { diff --git a/sw/common/nan_box.h b/sw/common/nan_box.h new file mode 100644 index 0000000000..4d68737a9b --- /dev/null +++ b/sw/common/nan_box.h @@ -0,0 +1,56 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +// RISC-V NaN-boxing: a narrow floating-point value held in a wider FP register +// has every register bit above it set to one; a value that is not boxed reads +// as the narrow format's canonical NaN. +namespace vortex::nanbox { + +constexpr uint64_t low_mask(uint32_t bits) { + return (bits >= 64) ? ~uint64_t(0) : ((uint64_t(1) << bits) - 1); +} + +constexpr uint64_t box(uint64_t value, uint32_t bits, uint32_t reg_bits = 64) { + return ((value & low_mask(bits)) | ~low_mask(bits)) & low_mask(reg_bits); +} + +constexpr bool is_boxed(uint64_t value, uint32_t bits, uint32_t reg_bits = 64) { + uint64_t upper = low_mask(reg_bits) & ~low_mask(bits); + return (value & upper) == upper; +} + +constexpr uint64_t unbox(uint64_t value, uint32_t bits, uint64_t nan, uint32_t reg_bits = 64) { + return is_boxed(value, bits, reg_bits) ? (value & low_mask(bits)) : nan; +} + +constexpr uint64_t box32(uint32_t value) { + return box(value, 32); +} + +constexpr uint32_t unbox32(uint64_t value, uint32_t nan) { + return uint32_t(unbox(value, 32, nan)); +} + +constexpr uint64_t box16(uint16_t value) { + return box(value, 16); +} + +constexpr uint16_t unbox16(uint64_t value, uint16_t nan) { + return uint16_t(unbox(value, 16, nan)); +} + +} diff --git a/sw/kernel/include/vx_mufu.h b/sw/kernel/include/vx_mufu.h index 5ce4480b81..7a236d64c1 100644 --- a/sw/kernel/include/vx_mufu.h +++ b/sw/kernel/include/vx_mufu.h @@ -27,13 +27,13 @@ extern "C" { #endif -#define __VX_MUFU_S(name, fn) \ -__attribute__((always_inline)) \ -inline float name(float x) { \ - float r; \ - __asm__ (".insn r %1, 0, 0, %0, %2, x" #fn \ - : "=f"(r) : "i"(RISCV_CUSTOM2), "f"(x)); \ - return r; \ +#define __VX_MUFU_S(name, fn) \ +__attribute__((always_inline)) \ +inline float name(float x) { \ + float r; \ + __asm__ (".insn r %1, 0, 0, %0, %2, x" #fn \ + : "=f"(r) : "i"(RISCV_CUSTOM2), "f"(x)); \ + return r; \ } __VX_MUFU_S(vx_rcpf, 0) // 1/x @@ -50,53 +50,61 @@ __VX_MUFU_S(vx_tanhf, 6) // tanh(x) __attribute__((always_inline)) inline float vx_expf_fast(float x) { - return vx_ex2f(x * 1.44269504088896341f); + return vx_ex2f(x * 1.44269504088896341f); } __attribute__((always_inline)) inline float vx_logf_fast(float x) { - return vx_lg2f(x) * 0.693147180559945309f; + return vx_lg2f(x) * 0.693147180559945309f; } __attribute__((always_inline)) inline float vx_sinf_fast(float x) { - return vx_sinpif(x * 0.318309886183790672f); + return vx_sinpif(x * 0.318309886183790672f); } __attribute__((always_inline)) inline float vx_cosf_fast(float x) { - return vx_cospif(x * 0.318309886183790672f); + return vx_cospif(x * 0.318309886183790672f); } __attribute__((always_inline)) inline float vx_sqrtf_fast(float x) { - // x * rsqrt(x) is 0 * inf = NaN at both ends. - float r = x * vx_rsqrtf(x); - return (x == 0.0f || x == __builtin_inff()) ? x : r; + // x * rsqrt(x) is 0 * inf at both ends, and MUFU flushes subnormal x to zero. + // The fixups use integer selects: a float select lowers to a branch, which + // is not divergence-safe. + float r = x * vx_rsqrtf(x); + uint32_t xb, rb; + __builtin_memcpy(&xb, &x, sizeof(xb)); + __builtin_memcpy(&rb, &r, sizeof(rb)); + rb = ((xb & 0x7f800000) == 0) ? (xb & 0x80000000) : rb; + rb = (xb == 0x7f800000) ? xb : rb; + __builtin_memcpy(&r, &rb, sizeof(r)); + return r; } __attribute__((always_inline)) inline float vx_divf_fast(float a, float b) { - return a * vx_rcpf(b); + return a * vx_rcpf(b); } // Activations. __attribute__((always_inline)) inline float vx_sigmoidf_fast(float x) { - return 0.5f + 0.5f * vx_tanhf(0.5f * x); + return 0.5f + 0.5f * vx_tanhf(0.5f * x); } __attribute__((always_inline)) inline float vx_siluf_fast(float x) { - return x * vx_sigmoidf_fast(x); + return x * vx_sigmoidf_fast(x); } // GELU, tanh form: 0.5x(1 + tanh(sqrt(2/pi)(x + 0.044715x^3))). __attribute__((always_inline)) inline float vx_geluf_fast(float x) { - float u = 0.797884560802865356f * x * (1.0f + 0.044715f * x * x); - return 0.5f * x * (1.0f + vx_tanhf(u)); + float u = 0.797884560802865356f * x * (1.0f + 0.044715f * x * x); + return 0.5f * x * (1.0f + vx_tanhf(u)); } #ifdef __cplusplus diff --git a/sw/runtime/common/perf.cpp b/sw/runtime/common/perf.cpp index f716430498..4abbdc869e 100644 --- a/sw/runtime/common/perf.cpp +++ b/sw/runtime/common/perf.cpp @@ -273,7 +273,6 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { const bool fpu_en = (isa_flags & VX_ISA_STD_F) != 0; const bool tcu_en = (isa_flags & VX_ISA_EXT_TCU) != 0; - const bool mufu_en = (isa_flags & VX_ISA_EXT_MUFU) != 0; const bool tex_en = (isa_flags & VX_ISA_EXT_TEX) != 0; const bool raster_en = (isa_flags & VX_ISA_EXT_RASTER) != 0; const bool om_en = (isa_flags & VX_ISA_EXT_OM) != 0; @@ -657,23 +656,6 @@ extern "C" vx_result_t vx_device_dump_perf(vx_device_h hdevice, FILE *stream) { tot_tbuf_stalls, tot_tbuf_cache_hits, tot_lmem_reads); } break; - case VX_DCR_MPM_CLASS_MUFU: { - if (!mufu_en) { - perf_print(stream, "MUFU not supported"); - break; - } - uint64_t tot_stalls = 0, tot_instrs = 0; - for (uint32_t core_id = 0; core_id < num_cores; ++core_id) { - uint64_t stalls = 0, instrs = 0; - CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_MUFU_STALLS, core_id, &stalls), { return err; }); - CHECK_ERR(vx_device_mpm_query(hdevice, mpm_class, VX_CSR_MPM_MUFU_INSTRS, core_id, &instrs), { return err; }); - perf_print_core(stream, core_id, "mufu: instrs=%" PRIu64 ", dispatch_stalls=%" PRIu64, instrs, stalls); - tot_stalls += stalls; - tot_instrs += instrs; - } - perf_print(stream, "mufu: total_instrs=%" PRIu64 ", total_dispatch_stalls=%" PRIu64, tot_instrs, tot_stalls); - } break; - case VX_DCR_MPM_CLASS_TEX: { if (!tex_en) { perf_print(stream, "TEX not supported"); diff --git a/tests/regression/mufu/kernel.cpp b/tests/regression/mufu/kernel.cpp index fb19b4067a..54e5760b02 100644 --- a/tests/regression/mufu/kernel.cpp +++ b/tests/regression/mufu/kernel.cpp @@ -4,38 +4,43 @@ template static void apply(const float* src, float* dst, uint32_t n, uint32_t repeat, F f) { - uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) - return; - float y = f(src[i]); - // Extra rounds chain each result into the next call; they lengthen runs for - // the SimX/RTL cycle-parity check while keeping the result reproducible. - for (uint32_t r = 1; r < repeat; ++r) - y = f(y); - dst[i] = y; + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) { + return; + } + float x = src[i]; + float y = f(x); + // Extra rounds recompute f(x) to lengthen the run; the empty asm statements + // hide x and keep each result live so the rounds cannot be folded. + for (uint32_t r = 1; r < repeat; ++r) { + __asm__ volatile ("" : "+f"(x)); + y = f(x); + __asm__ volatile ("" :: "f"(y)); + } + dst[i] = y; } __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto src = reinterpret_cast(arg->src_addr); - auto dst = reinterpret_cast(arg->dst_addr); - uint32_t n = arg->num_points; - uint32_t k = arg->repeat; - switch (arg->op) { - case OP_RCP: apply(src, dst, n, k, vx_rcpf); break; - case OP_RSQRT: apply(src, dst, n, k, vx_rsqrtf); break; - case OP_EX2: apply(src, dst, n, k, vx_ex2f); break; - case OP_LG2: apply(src, dst, n, k, vx_lg2f); break; - case OP_SINPI: apply(src, dst, n, k, vx_sinpif); break; - case OP_COSPI: apply(src, dst, n, k, vx_cospif); break; - case OP_TANH: apply(src, dst, n, k, vx_tanhf); break; - case OP_EXPF: apply(src, dst, n, k, vx_expf_fast); break; - case OP_LOGF: apply(src, dst, n, k, vx_logf_fast); break; - case OP_SINF: apply(src, dst, n, k, vx_sinf_fast); break; - case OP_COSF: apply(src, dst, n, k, vx_cosf_fast); break; - case OP_SQRTF: apply(src, dst, n, k, vx_sqrtf_fast); break; - case OP_SIGMOID: apply(src, dst, n, k, vx_sigmoidf_fast); break; - case OP_SILU: apply(src, dst, n, k, vx_siluf_fast); break; - case OP_GELU: apply(src, dst, n, k, vx_geluf_fast); break; - default: break; - } + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + uint32_t n = arg->num_points; + uint32_t k = arg->repeat; + switch (arg->op) { + case OP_RCP: apply(src, dst, n, k, vx_rcpf); break; + case OP_RSQRT: apply(src, dst, n, k, vx_rsqrtf); break; + case OP_EX2: apply(src, dst, n, k, vx_ex2f); break; + case OP_LG2: apply(src, dst, n, k, vx_lg2f); break; + case OP_SINPI: apply(src, dst, n, k, vx_sinpif); break; + case OP_COSPI: apply(src, dst, n, k, vx_cospif); break; + case OP_TANH: apply(src, dst, n, k, vx_tanhf); break; + case OP_EXPF: apply(src, dst, n, k, vx_expf_fast); break; + case OP_LOGF: apply(src, dst, n, k, vx_logf_fast); break; + case OP_SINF: apply(src, dst, n, k, vx_sinf_fast); break; + case OP_COSF: apply(src, dst, n, k, vx_cosf_fast); break; + case OP_SQRTF: apply(src, dst, n, k, vx_sqrtf_fast); break; + case OP_SIGMOID: apply(src, dst, n, k, vx_sigmoidf_fast); break; + case OP_SILU: apply(src, dst, n, k, vx_siluf_fast); break; + case OP_GELU: apply(src, dst, n, k, vx_geluf_fast); break; + default: break; + } } diff --git a/tests/regression/mufu/main.cpp b/tests/regression/mufu/main.cpp index 70aa31bf7b..337f12ebeb 100644 --- a/tests/regression/mufu/main.cpp +++ b/tests/regression/mufu/main.cpp @@ -21,6 +21,10 @@ /////////////////////////////////////////////////////////////////////////////// +static const uint32_t kSqrtSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x7f800000, +}; + static const char* kOpNames[OP_COUNT] = { "rcp", "rsqrt", "ex2", "lg2", "sinpi", "cospi", "tanh", "expf", "logf", "sinf", "cosf", "sqrtf", "sigmoid", "silu", "gelu" @@ -50,8 +54,9 @@ static void show_usage() { std::cout << "Vortex MUFU Test." << std::endl; std::cout << "Usage: [-k: kernel] [-n points] [-r repeat] [-t op] [-h: help]" << std::endl; std::cout << " ops:"; - for (auto name : kOpNames) + for (auto name : kOpNames) { std::cout << " " << name; + } std::cout << std::endl; } @@ -67,8 +72,9 @@ static void parse_args(int argc, char **argv) { break; case 't': for (int i = 0; i < OP_COUNT; ++i) { - if (strcmp(optarg, kOpNames[i]) == 0) + if (strcmp(optarg, kOpNames[i]) == 0) { selected_op = i; + } } if (selected_op < 0) { show_usage(); @@ -91,11 +97,21 @@ static void parse_args(int argc, char **argv) { void cleanup() { if (device) { - if (src_buffer) vx_buffer_release(src_buffer); - if (dst_buffer) vx_buffer_release(dst_buffer); - if (kernel) vx_kernel_release(kernel); - if (module_) vx_module_release(module_); - if (queue) vx_queue_release(queue); + if (src_buffer) { + vx_buffer_release(src_buffer); + } + if (dst_buffer) { + vx_buffer_release(dst_buffer); + } + if (kernel) { + vx_kernel_release(kernel); + } + if (module_) { + vx_module_release(module_); + } + if (queue) { + vx_queue_release(queue); + } vx_device_dump_perf(device, stdout); vx_device_release(device); } @@ -103,8 +119,17 @@ void cleanup() { /////////////////////////////////////////////////////////////////////////////// -static float as_float(uint32_t u) { float f; memcpy(&f, &u, 4); return f; } -static uint32_t as_bits(float f) { uint32_t u; memcpy(&u, &f, 4); return u; } +static float as_float(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static uint32_t as_bits(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} static float rand_range(std::mt19937& rng, float lo, float hi) { return std::uniform_real_distribution(lo, hi)(rng); @@ -118,8 +143,9 @@ static float rand_log(std::mt19937& rng, float lo, float hi, bool sign) { static float gen_input(int op, std::mt19937& rng) { // A quarter of raw-op inputs are arbitrary bit patterns. - if (op <= OP_TANH && (rng() & 3) == 0) + if (op <= OP_TANH && (rng() & 3) == 0) { return as_float(uint32_t(rng())); + } switch (op) { case OP_RCP: return rand_log(rng, -125, 125, true); case OP_RSQRT: return rand_log(rng, -125, 125, false); @@ -154,9 +180,9 @@ static bool raw_accurate(int op, float x, float y) { default: return true; } // Out of scope for the bound: specials, and results the FTZ/overflow rules decide. - if (!std::isnormal(x) || std::isnan(r) || std::isinf(r) - || std::fabs(r) >= 3.4028234663852886e38 || std::fabs(r) < 1.1754943508222875e-38) + if (!std::isnormal(x) || std::isnan(r) || std::isinf(r) || std::fabs(r) >= 3.4028234663852886e38 || std::fabs(r) < 1.1754943508222875e-38) { return true; + } double err = std::fabs((double)y - r); int ex; std::frexp(r, &ex); @@ -173,6 +199,11 @@ static bool raw_accurate(int op, float x, float y) { // Composite reference and tolerance; the composition rounds its prescale, so // the bound grows with |x| where the prescale is a multiply. static bool composite_accurate(int op, float x, float y) { + // Subnormal inputs flush to a zero of the same sign. + if (op == OP_SQRTF && !std::isnormal(x)) { + float ref = std::isinf(x) ? x : x * 0.0f; + return as_bits(y) == as_bits(ref); + } double xd = x, r, tol; switch (op) { case OP_EXPF: r = std::exp(xd); tol = (2e-7 + 1e-7 * std::fabs(xd)) * std::fabs(r); break; @@ -194,9 +225,19 @@ static bool composite_accurate(int op, float x, float y) { static int run_op(int op, std::mt19937& rng, uint64_t src_addr, uint64_t dst_addr) { std::vector h_src(num_points); std::vector h_dst(num_points); - size_t ns = (op <= OP_TANH) ? std::min(num_points, sizeof(kSpecials) / sizeof(kSpecials[0])) : 0; - for (uint32_t i = 0; i < num_points; ++i) - h_src[i] = (i < ns) ? as_float(kSpecials[i]) : gen_input(op, rng); + const uint32_t* specials = nullptr; + size_t ns = 0; + if (op <= OP_TANH) { + specials = kSpecials; + ns = sizeof(kSpecials) / sizeof(kSpecials[0]); + } else if (op == OP_SQRTF) { + specials = kSqrtSpecials; + ns = sizeof(kSqrtSpecials) / sizeof(kSqrtSpecials[0]); + } + ns = std::min(num_points, ns); + for (uint32_t i = 0; i < num_points; ++i) { + h_src[i] = (i < ns) ? as_float(specials[i]) : gen_input(op, rng); + } kernel_arg_t arg = {}; arg.op = op; @@ -233,18 +274,16 @@ static int run_op(int op, std::mt19937& rng, uint64_t src_addr, uint64_t dst_add bool ok; if (op <= OP_TANH) { uint32_t ref = vortex::mufu::eval(op, as_bits(x)); - for (uint32_t r = 1; r < repeat; ++r) - ref = vortex::mufu::eval(op, ref); - ok = (as_bits(y) == ref); - if (ok && repeat == 1) - ok = raw_accurate(op, x, y); - if (!ok && errors < 20) + ok = (as_bits(y) == ref) && raw_accurate(op, x, y); + if (!ok && errors < 20) { printf("*** error: %s[%u] x=%a (0x%08x): actual=0x%08x expected=0x%08x\n", kOpNames[op], i, x, as_bits(x), as_bits(y), ref); + } } else { - ok = (repeat > 1) || composite_accurate(op, x, y); - if (!ok && errors < 20) - printf("*** error: %s[%u] x=%a: actual=%a\n", kOpNames[op], i, x, y); + ok = composite_accurate(op, x, y); + if (!ok && errors < 20) { + printf("*** error: %s[%u] x=%a (0x%08x): actual=%a\n", kOpNames[op], i, x, as_bits(x), y); + } } errors += !ok; } @@ -283,8 +322,9 @@ int main(int argc, char *argv[]) { std::mt19937 rng(0x6d756675); int errors = 0; for (int op = 0; op < OP_COUNT; ++op) { - if (selected_op >= 0 && op != selected_op) + if (selected_op >= 0 && op != selected_op) { continue; + } errors += run_op(op, rng, src_addr, dst_addr); } diff --git a/tests/regression/zfhmin/kernel.cpp b/tests/regression/zfhmin/kernel.cpp index 11d14215e5..6cc5b9a8ae 100644 --- a/tests/regression/zfhmin/kernel.cpp +++ b/tests/regression/zfhmin/kernel.cpp @@ -159,14 +159,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { if (arg->mode == ZF_MODE_FLAGS) { int nt = vx_num_threads(); for (int l = 0; l < nt; ++l) { - if (vx_thread_id() == l && i < arg->count) + if (vx_thread_id() == l && i < arg->count) { flag_case(arg, i); + } } return; } - if (i >= arg->count) + if (i >= arg->count) { return; - __asm__ volatile("fsrm %0" :: "r"(arg->frm)); + } + __asm__ volatile("fsrm %0" ::"r"(arg->frm)); switch (arg->mode) { case ZF_MODE_NARROW_S: narrow_s(arg, i); break; case ZF_MODE_WIDEN_S: widen_s(arg, i); break; diff --git a/tests/regression/zfhmin/main.cpp b/tests/regression/zfhmin/main.cpp index e35c2cfdb7..93b99fe854 100644 --- a/tests/regression/zfhmin/main.cpp +++ b/tests/regression/zfhmin/main.cpp @@ -97,9 +97,10 @@ void read_aux(Buf& aux, void* out, uint64_t size) { } void report(const char* what, uint64_t input, uint32_t rm, uint64_t expected, uint64_t actual) { - if (errors < 32) + if (errors < 32) { std::printf("*** %s in=0x%llx rm=%u expected=0x%llx actual=0x%llx\n", what, (unsigned long long)input, rm, (unsigned long long)expected, (unsigned long long)actual); + } ++errors; } @@ -132,8 +133,9 @@ std::vector f32_inputs(std::mt19937& rng) { uint32_t e = 127 - 30 + (rng() % 50); v.push_back((rng() & 0x807fffff) | (e << 23)); } - for (uint32_t i = 0; i < num_random / 4; ++i) + for (uint32_t i = 0; i < num_random / 4; ++i) { v.push_back(rng()); + } return v; } @@ -149,8 +151,12 @@ std::vector f64_inputs(std::mt19937& rng) { for (int e = -27; e <= 17; ++e) { for (int k = 0; k < 8; ++k) { uint64_t man = ((uint64_t(rng()) << 32) | rng()) & 0xfffffffffffffull; - if (k & 1) man = (man & ~0x3ffffffffffull) | 0x20000000000ull; - if (k & 2) man &= ~0x3ffffffffffull; + if (k & 1) { + man = (man & ~0x3ffffffffffull) | 0x20000000000ull; + } + if (k & 2) { + man &= ~0x3ffffffffffull; + } v.push_back((uint64_t(k & 4 ? 1 : 0) << 63) | (uint64_t(e + 1023) << 52) | man); } } @@ -172,19 +178,22 @@ void test_narrow_s(const std::vector& in) { read_aux(aux, mvx.data(), aux.size); for (uint32_t i = 0; i < n && i < verbose; ++i) { std::printf("in=0x%08x", in[i]); - for (uint32_t k = 0; k < ZF_NUM_RM; ++k) + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { std::printf(" %04x/%04x", out[i * ZF_NUM_RM + k], rv_ftoh_s(in[i], rm_of(k), nullptr)); + } std::printf("\n"); } for (uint32_t i = 0; i < n; ++i) { for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { uint16_t ref = rv_ftoh_s(in[i], rm_of(k), nullptr); - if (out[i * ZF_NUM_RM + k] != ref) + if (out[i * ZF_NUM_RM + k] != ref) { report("fcvt.h.s", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } } int64_t ref = sext16(rv_ftoh_s(in[i], 0, nullptr)); - if (mvx[i] != ref) + if (mvx[i] != ref) { report("fmv.x.h", in[i], 0, ref, mvx[i]); + } } std::printf("fcvt.h.s: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); } @@ -192,15 +201,18 @@ void test_narrow_s(const std::vector& in) { void test_widen_s() { uint32_t n = 65536; std::vector in(n); - for (uint32_t i = 0; i < n; ++i) in[i] = i; + for (uint32_t i = 0; i < n; ++i) { + in[i] = i; + } Buf src(n * 2), dst(n * 4); src.write(in.data()); std::vector out(n); run(ZF_MODE_WIDEN_S, n, src, dst, nullptr, out.data(), dst.size); for (uint32_t i = 0; i < n; ++i) { uint32_t ref = rv_htof_s(in[i], 0, nullptr); - if (out[i] != ref) + if (out[i] != ref) { report("fcvt.s.h", in[i], 0, ref, out[i]); + } } std::printf("fcvt.s.h: all %u half inputs\n", n); } @@ -214,8 +226,9 @@ void test_narrow_d(const std::vector& in) { for (uint32_t i = 0; i < n; ++i) { for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { uint16_t ref = rv_dtoh(in[i], rm_of(k), nullptr); - if (out[i * ZF_NUM_RM + k] != ref) + if (out[i * ZF_NUM_RM + k] != ref) { report("fcvt.h.d", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } } } std::printf("fcvt.h.d: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); @@ -224,34 +237,43 @@ void test_narrow_d(const std::vector& in) { void test_widen_d() { uint32_t n = 65536; std::vector in(n); - for (uint32_t i = 0; i < n; ++i) in[i] = i; + for (uint32_t i = 0; i < n; ++i) { + in[i] = i; + } Buf src(n * 2), dst(n * 8); src.write(in.data()); std::vector out(n); run(ZF_MODE_WIDEN_D, n, src, dst, nullptr, out.data(), dst.size); for (uint32_t i = 0; i < n; ++i) { uint64_t ref = rv_htod(in[i], 0, nullptr); - if (out[i] != ref) + if (out[i] != ref) { report("fcvt.d.h", in[i], 0, ref, out[i]); + } } std::printf("fcvt.d.h: all %u half inputs\n", n); } void test_flags(const std::vector& f32, const std::vector& f64) { std::vector cases; - for (size_t i = 0; i < num_flag_in && i < f32.size(); ++i) - for (uint32_t rm = 0; rm < 5; ++rm) + for (size_t i = 0; i < num_flag_in && i < f32.size(); ++i) { + for (uint32_t rm = 0; rm < 5; ++rm) { cases.push_back({ZF_OP_H_S, rm, f32[i]}); + } + } const uint16_t halves[] = {0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0x7bff, 0x7c00, 0xfc00, 0x7e00, 0x7c01, 0x7d00, 0xfe00, 0xfd55, 0x7fff, 0x1234}; - for (uint16_t h : halves) + for (uint16_t h : halves) { cases.push_back({ZF_OP_S_H, 0, h}); + } if (xlen == 64) { - for (size_t i = 0; i < 64 && i < f64.size(); ++i) - for (uint32_t rm = 0; rm < 5; ++rm) + for (size_t i = 0; i < 64 && i < f64.size(); ++i) { + for (uint32_t rm = 0; rm < 5; ++rm) { cases.push_back({ZF_OP_H_D, rm, f64[i]}); - for (uint16_t h : halves) + } + } + for (uint16_t h : halves) { cases.push_back({ZF_OP_D_H, 0, h}); + } } uint32_t n = cases.size(); Buf src(n * sizeof(zf_flag_case_t)), dst(n * sizeof(zf_flag_out_t)); @@ -269,8 +291,9 @@ void test_flags(const std::vector& f32, const std::vector& f case ZF_OP_H_D: ref = rv_dtoh(c.bits, c.rm, &fl); break; case ZF_OP_D_H: ref = rv_htod(uint16_t(c.bits), c.rm, &fl); break; } - if (out[i].result != ref) + if (out[i].result != ref) { report(names[c.op], c.bits, c.rm, ref, out[i].result); + } if (out[i].fflags != fl) { char what[32]; std::snprintf(what, sizeof(what), "%s fflags", names[c.op]); @@ -287,8 +310,12 @@ void test_box(std::mt19937& rng) { for (uint32_t i = 0; i < n; ++i) { h[i] = rng(); g[i] = rng(); - if (i % 4 == 0) g[i] |= 0xffff0000; - if (i % 8 == 1) g[i] &= 0x0000ffff; + if (i % 4 == 0) { + g[i] |= 0xffff0000; + } + if (i % 8 == 1) { + g[i] &= 0x0000ffff; + } } Buf src(n * 2), aux(n * 4), dst(n * ZF_BOX_OUTS * 8); src.write(h.data()); @@ -303,11 +330,21 @@ void test_box(std::mt19937& rng) { uint64_t r2 = (uint64_t)sext16(h[i]); uint64_t r3 = ((g[i] >> 16) == 0xffff) ? rv_htof_s(uint16_t(g[i]), 0, nullptr) : 0x7fc00000u; uint64_t r4 = (uint64_t)sext16(g[i]); - if (o[0] != r0) report("flh box", h[i], 0, r0, o[0]); - if (o[1] != r1) report("fmv.h.x box", g[i], 0, r1, o[1]); - if (o[2] != r2) report("flh+fmv.x.h", h[i], 0, r2, o[2]); - if (o[3] != r3) report("unboxed fcvt.s.h", g[i], 0, r3, o[3]); - if (o[4] != r4) report("fmv.x.h unboxed", g[i], 0, r4, o[4]); + if (o[0] != r0) { + report("flh box", h[i], 0, r0, o[0]); + } + if (o[1] != r1) { + report("fmv.h.x box", g[i], 0, r1, o[1]); + } + if (o[2] != r2) { + report("flh+fmv.x.h", h[i], 0, r2, o[2]); + } + if (o[3] != r3) { + report("unboxed fcvt.s.h", g[i], 0, r3, o[3]); + } + if (o[4] != r4) { + report("fmv.x.h unboxed", g[i], 0, r4, o[4]); + } } std::printf("boxing: %u cases\n", n); } @@ -336,14 +373,26 @@ int main(int argc, char** argv) { std::printf("zfhmin: xlen=%u dynamic frm=%u\n", xlen, frm_dyn); auto f32 = f32_inputs(rng); auto f64 = f64_inputs(rng); - if (phases & 0x01) test_narrow_s(f32); - if (phases & 0x02) test_widen_s(); + if (phases & 0x01) { + test_narrow_s(f32); + } + if (phases & 0x02) { + test_widen_s(); + } if (xlen == 64) { - if (phases & 0x04) test_narrow_d(f64); - if (phases & 0x08) test_widen_d(); + if (phases & 0x04) { + test_narrow_d(f64); + } + if (phases & 0x08) { + test_widen_d(); + } + } + if (phases & 0x10) { + test_flags(f32, f64); + } + if (phases & 0x20) { + test_box(rng); } - if (phases & 0x10) test_flags(f32, f64); - if (phases & 0x20) test_box(rng); vx_kernel_release(kern); vx_module_release(mod); From e77ac8eb384d96df3c543fb6dfb475e143672056 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 23 Oct 2026 01:41:49 -0700 Subject: [PATCH 8/9] mufu: hand results to the SFU's response buffer The SFU's response switch already buffers each PE's results, so the MUFU serializer's own two-slot output buffer was a second copy of a full 16-lane result. Removing it drops MUFU's result latency by one cycle (SimX matches) with no throughput loss, including with one MUFU lane per SFU lane. Re-recorded mufu baselines (fpga_gate/asic_gate --update-baseline): Xilinx 2030 LUT / 1105 FF (was 2624 / 2278), Yosys 2073 um^2 (was 2701). --- ci/baselines/synthesis/xilinx/mufu.json | 166 ++++++++++++------------ ci/baselines/synthesis/yosys/mufu.json | 14 +- hw/rtl/mufu/VX_mufu_unit.sv | 2 +- sim/simx/mufu_unit.h | 5 +- 4 files changed, 93 insertions(+), 94 deletions(-) diff --git a/ci/baselines/synthesis/xilinx/mufu.json b/ci/baselines/synthesis/xilinx/mufu.json index 2e4c0076a4..1407de459e 100644 --- a/ci/baselines/synthesis/xilinx/mufu.json +++ b/ci/baselines/synthesis/xilinx/mufu.json @@ -6,107 +6,107 @@ "dut": "mufu", "result": { "bram": 1, - "build_time_s": 796, + "build_time_s": 712, "critical_paths": [ { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", "levels": 12, - "logic_ns": 1.126, - "route_ns": 1.709, - "route_pct": 60.288, - "slack_ns": 0.381, + "logic_ns": 1.33, + "route_ns": 1.795, + "route_pct": 57.443, + "slack_ns": 0.187, "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", "levels": 12, - "logic_ns": 1.127, - "route_ns": 1.707, - "route_pct": 60.239, - "slack_ns": 0.382, + "logic_ns": 1.33, + "route_ns": 1.792, + "route_pct": 57.397, + "slack_ns": 0.191, "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", "levels": 12, - "logic_ns": 1.128, - "route_ns": 1.705, - "route_pct": 60.18, - "slack_ns": 0.383, - "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" + "logic_ns": 1.328, + "route_ns": 1.793, + "route_pct": 57.452, + "slack_ns": 0.191, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", "levels": 12, - "logic_ns": 1.126, - "route_ns": 1.706, - "route_pct": 60.246, - "slack_ns": 0.384, + "logic_ns": 1.329, + "route_ns": 1.791, + "route_pct": 57.402, + "slack_ns": 0.193, "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.126, - "route_ns": 1.706, - "route_pct": 60.246, - "slack_ns": 0.384, - "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + "levels": 11, + "logic_ns": 1.225, + "route_ns": 1.879, + "route_pct": 60.54, + "slack_ns": 0.209, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[3]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.127, - "route_ns": 1.704, - "route_pct": 60.197, - "slack_ns": 0.385, + "levels": 13, + "logic_ns": 1.333, + "route_ns": 1.739, + "route_pct": 56.615, + "slack_ns": 0.24, "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.127, - "route_ns": 1.704, - "route_pct": 60.197, - "slack_ns": 0.385, + "levels": 13, + "logic_ns": 1.335, + "route_ns": 1.737, + "route_pct": 56.545, + "slack_ns": 0.24, "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][77]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.128, - "route_ns": 1.702, - "route_pct": 60.138, - "slack_ns": 0.386, - "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" + "levels": 13, + "logic_ns": 1.334, + "route_ns": 1.737, + "route_pct": 56.568, + "slack_ns": 0.241, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][79]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.128, - "route_ns": 1.702, - "route_pct": 60.138, - "slack_ns": 0.386, - "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" + "levels": 13, + "logic_ns": 1.332, + "route_ns": 1.733, + "route_pct": 56.544, + "slack_ns": 0.247, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" }, { - "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][80]/S", + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", "group": "core_clock", - "levels": 12, - "logic_ns": 1.125, - "route_ns": 1.703, - "route_pct": 60.216, - "slack_ns": 0.388, - "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[0]/C" + "levels": 13, + "logic_ns": 1.31, + "route_ns": 1.752, + "route_pct": 57.215, + "slack_ns": 0.251, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" } ], "dsp": 3, @@ -116,28 +116,23 @@ "vivado": "2024.2", "xlen": 32 }, - "ff": 2278, - "fmax_mhz": 338.8, + "ff": 1105, + "fmax_mhz": 317.9, "high_fanout_nets": [ { "driver": "FDRE", - "fanout": 1556, + "fanout": 1044, "net": "mufu_unit/pe_serializer/shift_reg/pe_valid_in" }, { "driver": "LUT2", "fanout": 718, - "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep_0" + "net": "mufu_unit/pe_serializer/pe_enable" }, { - "driver": "LUT2", - "fanout": 583, - "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.fire_in" - }, - { - "driver": "LUT2", - "fanout": 583, - "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.flow_out" + "driver": "FDRE", + "fanout": 165, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx_reg[3]_rep_n_0" }, { "driver": "MUXF8", @@ -151,7 +146,7 @@ }, { "driver": "FDRE", - "fanout": 162, + "fanout": 154, "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx[3]" }, { @@ -160,20 +155,25 @@ "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[25]" }, { - "driver": "FDSE", - "fanout": 146, - "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep_n_0" + "driver": "FDRE", + "fanout": 133, + "net": "mufu_unit/pe_serializer/g_serialize.batch_in_idx[0]" }, { - "driver": "FDSE", - "fanout": 143, - "net": "mufu_unit/pe_serializer/out_buf/g_eb2.stream_buffer/g_buffer.valid_in_r_reg_rep__3_n_0" + "driver": "FDRE", + "fanout": 132, + "net": "mufu_unit/pe_serializer/g_serialize.batch_in_idx[1]" + }, + { + "driver": "FDRE", + "fanout": 131, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx[1]" } ], - "lut": 2624, + "lut": 2030, "lutram": 64, "uram": 0, - "wns_ns": 0.381 + "wns_ns": 0.187 } } } diff --git a/ci/baselines/synthesis/yosys/mufu.json b/ci/baselines/synthesis/yosys/mufu.json index e9740effe8..ee52210d5f 100644 --- a/ci/baselines/synthesis/yosys/mufu.json +++ b/ci/baselines/synthesis/yosys/mufu.json @@ -5,9 +5,9 @@ "configs": "", "dut": "mufu", "result": { - "build_time_s": 130, - "cell_area_um2": 2700.974, - "cell_count": 26212, + "build_time_s": 195, + "cell_area_um2": 2073.466, + "cell_count": 20518, "env": { "corner": "tt", "pdk": "asap7", @@ -17,12 +17,12 @@ "xlen": 32, "yosys": "Yosys 0.69" }, - "fmax_mhz": 1069.519, - "power_mw": 19.1, - "seq_area_um2": 847.973, + "fmax_mhz": 1039.501, + "power_mw": 17.3, + "seq_area_um2": 507.384, "sram_area_um2": 0.0, "tns_ns": 0.0, - "wns_ns": 0.315 + "wns_ns": 0.288 } } } diff --git a/hw/rtl/mufu/VX_mufu_unit.sv b/hw/rtl/mufu/VX_mufu_unit.sv index 394f9fd17d..93109a1787 100644 --- a/hw/rtl/mufu/VX_mufu_unit.sv +++ b/hw/rtl/mufu/VX_mufu_unit.sv @@ -48,7 +48,7 @@ module VX_mufu_unit import VX_gpu_pkg::*; #( .SHARED_WIDTH (INST_MUFU_BITS), .TAG_WIDTH (HDR_WIDTH), .PE_REG (0), - .OUT_BUF (3) + .OUT_BUF (0) ) pe_serializer ( .clk (clk), .reset (reset), diff --git a/sim/simx/mufu_unit.h b/sim/simx/mufu_unit.h index 510f122669..6e4e780c1d 100644 --- a/sim/simx/mufu_unit.h +++ b/sim/simx/mufu_unit.h @@ -24,9 +24,8 @@ namespace vortex { // datapath lanes, one batch per cycle, and a warp is accepted on its last batch. class MufuUnit { public: - // Cycles a result spends in the datapath and its registered output after the - // warp is accepted. - static constexpr uint32_t kLatency = VX_MUFU_LATENCY + 1; + // Cycles a result spends in the datapath after the warp is accepted. + static constexpr uint32_t kLatency = VX_MUFU_LATENCY; // Returns false while the warp's batches are still being issued; on the // accepting cycle computes the results and returns true. From c8f5980489dcf5451012f3cfa6887ba32f5170e3 Mon Sep 17 00:00:00 2001 From: jel221 Date: Fri, 23 Oct 2026 04:15:39 -0700 Subject: [PATCH 9/9] syn: give the xilinx fpu DUT VX_tlb_pkg The xilinx DUT flow compiles every file on the include path, so the fpu DUT's BASE_INC brings in hw/rtl/VX_cluster.sv, which imports VX_tlb_pkg. fpu_PKG never listed the package, and synthesis stops with: ERROR: [Synth 8-36] 'VX_tlb_pkg' is not declared [.../VX_cluster.sv:1] The mufu DUT lists it for the same reason. --- hw/syn/xilinx/dut/catalog.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index ff5cb1b333..9a2cde4690 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -61,6 +61,8 @@ dxa_PKG = $(RTL_DIR)/dxa/VX_dxa_pkg.sv fpu_IP := 1 fpu_PRJ = $(if $(filter -DVX_CFG_FPU_TYPE_FPNEW,$(XCONFIGS)),VX_fpu_fpnew,$(if $(filter -DVX_CFG_FPU_TYPE_STD,$(XCONFIGS)),VX_fpu_std,VX_fpu_dsp)) fpu_INC = $(BASE_INC) $(FPU_INC) +# BASE_INC pulls in hw/rtl/VX_cluster.sv, which imports VX_tlb_pkg. +fpu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv mufu_PRJ := VX_mufu_unit_top mufu_CFG := -DVX_CFG_EXT_MUFU_ENABLE