diff --git a/VX_config.toml b/VX_config.toml index f51186cdec..ca184b4886 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -34,6 +34,9 @@ VX_CFG_EXT_A_ENABLE = false VX_CFG_EXT_ZACAS_ENABLE = false VX_CFG_EXT_ZICOND_ENABLE = true +# Zfhmin: half-precision load/store, move and fp16<->fp32/fp64 conversion (no +# half arithmetic). Requires F. +VX_CFG_EXT_ZFHMIN_ENABLE = false VX_CFG_EXT_TCU_ENABLE = false VX_CFG_EXT_DMA_ENABLE = false VX_CFG_EXT_DXA_ENABLE = false @@ -41,6 +44,7 @@ VX_CFG_EXT_TEX_ENABLE = false VX_CFG_EXT_RASTER_ENABLE = false VX_CFG_EXT_OM_ENABLE = false VX_CFG_EXT_RTU_ENABLE = false +VX_CFG_EXT_MUFU_ENABLE = false VX_CFG_VLEN = "expr: $VX_CFG_XLEN * 4" [pipeline] @@ -149,6 +153,10 @@ VX_CFG_FSQRT_PE_RATIO= 1 VX_CFG_FCVT_PE_RATIO = 1 VX_CFG_FNCP_PE_RATIO = 1 +[mufu] +# MUFU datapath lanes inside the SFU; the SFU's lanes are serialized onto them. +VX_CFG_NUM_MUFU_LANES = 1 + [amo] # LR/SC reservation stations per LLC bank; line-indexed in BRAM, eviction yields legal spurious SC failures VX_CFG_AMO_RS_SIZE = "expr: max($VX_CFG_NUM_WARPS, up($VX_CFG_NUM_CORES * $VX_CFG_NUM_WARPS / $VX_CFG_L2_NUM_BANKS))" @@ -447,7 +455,7 @@ VX_CFG_NUM_OCACHES = "expr: max(1, up($VX_CFG_NUM_OM_CORES / 4))" VX_CFG_MISA_STD = "expr: ($VX_CFG_EXT_A_ENABLED << 0) | (0 << 1) | ($VX_CFG_EXT_C_ENABLED << 2) | ($VX_CFG_EXT_D_ENABLED << 3) | (0 << 4) | ($VX_CFG_EXT_F_ENABLED << 5) | (0 << 6) | (0 << 7) | (1 << 8) | (0 << 9) | (0 << 10) | (0 << 11) | ($VX_CFG_EXT_M_ENABLED << 12) | (0 << 13) | (0 << 14) | (0 << 15) | (0 << 16) | (0 << 17) | (0 << 18) | (0 << 19) | (1 << 20) | (0 << 21) | (0 << 22) | (1 << 23) | (0 << 24) | (0 << 25)" # Custom extensions signature -VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11)" +VX_CFG_MISA_EXT = "expr: ($VX_CFG_ICACHE_ENABLED << 0) | ($VX_CFG_DCACHE_ENABLED << 1) | ($VX_CFG_L2_ENABLED << 2) | ($VX_CFG_L3_ENABLED << 3) | ($VX_CFG_LMEM_ENABLED << 4) | ($VX_CFG_EXT_ZICOND_ENABLED << 5) | ($VX_CFG_EXT_TCU_ENABLED << 9) | ($VX_CFG_EXT_DXA_ENABLED << 10) | ($VX_CFG_EXT_TEX_ENABLED << 6) | ($VX_CFG_EXT_RASTER_ENABLED << 7) | ($VX_CFG_EXT_OM_ENABLED << 8) | ($VX_CFG_EXT_RTU_ENABLED << 11) | ($VX_CFG_EXT_MUFU_ENABLED << 12) | ($VX_CFG_EXT_ZFHMIN_ENABLED << 13)" [toolchain] ASIC = false diff --git a/VX_types.toml b/VX_types.toml index 7143523976..ff3437215e 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -56,6 +56,9 @@ VX_MEM_IO_END_ADDR = "expr: $VX_MEM_USER_BASE_ADDR" VX_MEM_OM_BASE_ADDR = "expr: 0x00000000E0000000 if ($XLEN == 64) else 0xE0000000" VX_MEM_OM_END_ADDR = "expr: $VX_MEM_PAGE_TABLE_BASE_ADDR" +[mufu] +VX_MUFU_LATENCY = 7 + # VM page-table format — RISC-V architectural constants fixed by the SATP mode # (SV32 vs SV39, selected by XLEN). Relocated from VX_config.toml. [vm] diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 77e8c4785a..8467f80317 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -6,7 +6,7 @@ }, "app": "raycast", "args": "", - "config_hash": "396cee9df6025cdb", + "config_hash": "30868719eb39dd56", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "raycast", "args": "", - "config_hash": "262f24032ef14b58", + "config_hash": "bbb5e759fb64a2d2", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "ad2b37492d761e8f", + "config_hash": "f3cdbb1a54b60bd7", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "606d011dc6f612d3", + "config_hash": "1f13c513ce419a18", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "9390754d3719fca3", + "config_hash": "6c4c16bd5ba783fb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm", "args": "-n128", - "config_hash": "740fd84fa2938103", + "config_hash": "e654782241f3196f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "2e1c44e05f9ac34d", + "config_hash": "406f5c17ca35ee9d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemmx", "args": "-n128", - "config_hash": "024adb49ed96bdd3", + "config_hash": "5016dd4b1102f6fc", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "a3d9df9695acbb95", + "config_hash": "b631a0bafc91d264", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "0e8990b25e64ad7f", + "config_hash": "14333df532bad1d6", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "9e0c1c9767c760ea", + "config_hash": "fdbcac76ee256139", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -127,7 +127,7 @@ }, "app": "softmax", "args": "-n64", - "config_hash": "eb5fed4bfce19d76", + "config_hash": "d1489b9fccb46500", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -138,7 +138,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "6925a5138be930ea", + "config_hash": "7514f2e770fda7af", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -149,7 +149,7 @@ }, "app": "stencil3d", "args": "-n32", - "config_hash": "b83a1e3735f79d39", + "config_hash": "7b1283f03793abb6", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -160,7 +160,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ef0c2548e7c5bf86", + "config_hash": "3c79fc19ad8549a0", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -171,7 +171,7 @@ }, "app": "vecadd", "args": "-n16384", - "config_hash": "ce1eb79984a98552", + "config_hash": "75b65abb22c3753b", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index bf1e1a010c..a57e2798f3 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -6,7 +6,7 @@ }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "6a815f855f6e0d05", + "config_hash": "1d98c347f39baa17", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "626c5f6244584b1f", + "config_hash": "f0ace578558d01a1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "7c32a11d123bfb13", + "config_hash": "9a3f1b02710f7458", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "822d0ca194266a27", + "config_hash": "4d3b7f000ce31563", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "87ea544e77719254", + "config_hash": "4dffe3629adfe705", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "e55b61598f5f10ea", + "config_hash": "6aac48586a19e5d1", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "21da2f2737bf03dd", + "config_hash": "15066287b6cf674a", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "20304652a1bfb94c", + "config_hash": "6f3f943387268cd9", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "bf4520d008c07dc9", + "config_hash": "99f4567a3f3d04a6", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -105,7 +105,7 @@ }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "2a4e017d98740636", + "config_hash": "a45156b86aef0add", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, @@ -116,7 +116,7 @@ }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "f05069d7cf0ee3ff", + "config_hash": "63afe161fb9d2aac", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index b5ee86bfc2..761b756cb9 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -6,7 +6,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "1e2937b20579ad3f", + "config_hash": "71e067fd7765f075", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "edc924bc4f246971", + "config_hash": "ac94e8341d7c171e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "d682cc8c6f99c06b", + "config_hash": "00c3aa3428bf4e86", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "46a66220c1d9956b", + "config_hash": "a99eb5442dfe8bd5", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "a7554f6fa88474dc", + "config_hash": "b1631e4a43822c5a", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -61,7 +61,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "e2e1879d7d6f9aaf", + "config_hash": "2c54468d2adf668e", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -72,7 +72,7 @@ }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "6e48f6b249533cc5", + "config_hash": "1f50896f410bc97f", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, @@ -83,7 +83,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "c8c6b30537e93e2f", + "config_hash": "b72e550c7f6c7aca", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -94,7 +94,7 @@ }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "a4fc4d97e6e80586", + "config_hash": "15f81408ca91fc9f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 1fb0ab7f7b..db73f213c5 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -6,7 +6,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "9a4f28d2555289b5", + "config_hash": "048a348462ce8158", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "aef578263250d585", + "config_hash": "bbab935bba11ff85", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "rt_raycast", "args": "", - "config_hash": "c7b45fe5c1828bc5", + "config_hash": "f2e45a7a7018b151", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3552c4620b..85797034f5 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "da0a50e46103b7dc", + "config_hash": "d538172b26e2967c", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "2d12f1a0058fcc3b", + "config_hash": "02375c8f3588b56d", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "fce886092e60e0eb", + "config_hash": "38834e790f4f6a3f", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, @@ -39,7 +39,7 @@ }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "f0558a3b3dfde51f", + "config_hash": "fcf4a952a5f93107", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, @@ -50,7 +50,7 @@ }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "24fd7a811fabb8c4", + "config_hash": "66b2e78e9663c54b", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index 1dc1b78f41..a9ed6b1e98 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "3cfc301023342a7d", + "config_hash": "eac1b740108b63ac", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index 2c4f92004c..d898fae929 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "90defe3d20b6f739", + "config_hash": "1ab2d7ff293df75a", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index c7f771c72f..1caa93120b 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -6,7 +6,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "ba81956fe68e94b6", + "config_hash": "23715dbc37d808e1", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, @@ -17,7 +17,7 @@ }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "6ee14f25ec573117", + "config_hash": "3eafd3c24fc1aba9", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, @@ -28,7 +28,7 @@ }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "a9ea3d8bd83a7eed", + "config_hash": "448955a5824dd803", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/synthesis/xilinx/mufu.json b/ci/baselines/synthesis/xilinx/mufu.json new file mode 100644 index 0000000000..1407de459e --- /dev/null +++ b/ci/baselines/synthesis/xilinx/mufu.json @@ -0,0 +1,179 @@ +{ + "mufu": { + "clock_mhz": 300, + "config_hash": "5d5eb85429e40b51", + "configs": "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16", + "dut": "mufu", + "result": { + "bram": 1, + "build_time_s": 712, + "critical_paths": [ + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 12, + "logic_ns": 1.33, + "route_ns": 1.795, + "route_pct": 57.443, + "slack_ns": 0.187, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 12, + "logic_ns": 1.33, + "route_ns": 1.792, + "route_pct": 57.397, + "slack_ns": 0.191, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 12, + "logic_ns": 1.328, + "route_ns": 1.793, + "route_pct": 57.452, + "slack_ns": 0.191, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 12, + "logic_ns": 1.329, + "route_ns": 1.791, + "route_pct": 57.402, + "slack_ns": 0.193, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 11, + "logic_ns": 1.225, + "route_ns": 1.879, + "route_pct": 60.54, + "slack_ns": 0.209, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[3]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 13, + "logic_ns": 1.333, + "route_ns": 1.739, + "route_pct": 56.615, + "slack_ns": 0.24, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 13, + "logic_ns": 1.335, + "route_ns": 1.737, + "route_pct": 56.545, + "slack_ns": 0.24, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 13, + "logic_ns": 1.334, + "route_ns": 1.737, + "route_pct": 56.568, + "slack_ns": 0.241, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 13, + "logic_ns": 1.332, + "route_ns": 1.733, + "route_pct": 56.544, + "slack_ns": 0.247, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + }, + { + "endpoint": "mufu_unit/g_pes[0].pe/s1_reg/g_pipe.g_no_reset.pipe_reg[0][16]/D", + "group": "core_clock", + "levels": 13, + "logic_ns": 1.31, + "route_ns": 1.752, + "route_pct": 57.215, + "slack_ns": 0.251, + "startpoint": "mufu_unit/pe_serializer/g_serialize.batch_in_idx_reg[1]/C" + } + ], + "dsp": 3, + "env": { + "device": "xcu55c-fsvh2892-2L-e", + "opt_level": 3, + "vivado": "2024.2", + "xlen": 32 + }, + "ff": 1105, + "fmax_mhz": 317.9, + "high_fanout_nets": [ + { + "driver": "FDRE", + "fanout": 1044, + "net": "mufu_unit/pe_serializer/shift_reg/pe_valid_in" + }, + { + "driver": "LUT2", + "fanout": 718, + "net": "mufu_unit/pe_serializer/pe_enable" + }, + { + "driver": "FDRE", + "fanout": 165, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx_reg[3]_rep_n_0" + }, + { + "driver": "MUXF8", + "fanout": 164, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[24]" + }, + { + "driver": "MUXF8", + "fanout": 162, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[23]" + }, + { + "driver": "FDRE", + "fanout": 154, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx[3]" + }, + { + "driver": "MUXF8", + "fanout": 148, + "net": "mufu_unit/pe_serializer/pe_data_in[0]_39[25]" + }, + { + "driver": "FDRE", + "fanout": 133, + "net": "mufu_unit/pe_serializer/g_serialize.batch_in_idx[0]" + }, + { + "driver": "FDRE", + "fanout": 132, + "net": "mufu_unit/pe_serializer/g_serialize.batch_in_idx[1]" + }, + { + "driver": "FDRE", + "fanout": 131, + "net": "mufu_unit/pe_serializer/g_serialize.batch_out_idx[1]" + } + ], + "lut": 2030, + "lutram": 64, + "uram": 0, + "wns_ns": 0.187 + } + } +} diff --git a/ci/baselines/synthesis/yosys/mufu.json b/ci/baselines/synthesis/yosys/mufu.json new file mode 100644 index 0000000000..ee52210d5f --- /dev/null +++ b/ci/baselines/synthesis/yosys/mufu.json @@ -0,0 +1,28 @@ +{ + "mufu": { + "clock_mhz": 800, + "config_hash": "ec81347c2979f17c", + "configs": "", + "dut": "mufu", + "result": { + "build_time_s": 195, + "cell_area_um2": 2073.466, + "cell_count": 20518, + "env": { + "corner": "tt", + "pdk": "asap7", + "sta": "3.1.0", + "sv2v": "sv2v v0.0.13-13-g493a88f", + "vt": "rvt", + "xlen": 32, + "yosys": "Yosys 0.69" + }, + "fmax_mhz": 1039.501, + "power_mw": 17.3, + "seq_area_um2": 507.384, + "sram_area_um2": 0.0, + "tns_ns": 0.0, + "wns_ns": 0.288 + } + } +} diff --git a/ci/testcases/asic_gate.yaml b/ci/testcases/asic_gate.yaml index 5f81318062..00def8e7cc 100644 --- a/ci/testcases/asic_gate.yaml +++ b/ci/testcases/asic_gate.yaml @@ -136,6 +136,16 @@ builds: clock_mhz: 800 configs: "" +# --- mufu -------------------------------------------------------------------- + +# Special-function datapath at its default single lane, serialized from a +# 16-lane SFU: the 7-stage table+polynomial pipeline and the 512x72 table ROM. +- id: mufu + group: mufu + dut: mufu + clock_mhz: 800 + configs: "" + # --- tensor ------------------------------------------------------------------ # TFR tensor core with every datatype plus sparsity and WGMMA. The widest diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index aba38f15d9..cae9b0f0a8 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -103,6 +103,16 @@ builds: clock_mhz: 300 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# --- mufu -------------------------------------------------------------------- + +# Special-function datapath at its default single lane, serialized from a +# 16-lane SFU: the 7-stage table+polynomial pipeline and the 512x72 table ROM. +- id: mufu + group: mufu + dut: mufu + clock_mhz: 300 + configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16" + # --- tensor ------------------------------------------------------------------ # TFR tensor core with every datatype plus sparsity and WGMMA. Widest FEDP/DSP48 diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml new file mode 100644 index 0000000000..c7f70873b9 --- /dev/null +++ b/ci/testcases/mufu.yaml @@ -0,0 +1,64 @@ +# MUFU special-function unit (custom-2: rcp/rsqrt/ex2/lg2/sinpi/cospi/tanh). +# The `mufu` app self-enables VX_CFG_EXT_MUFU_ENABLE and checks every result +# bit-exactly against the shared golden model (sw/common/mufu.cpp) plus libm +# accuracy bounds; the datapath itself is covered by unittest:hw-mufu. +category: mufu +defaults: + xlen: + - 32 + - 64 + tier: smoke + touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] +tests: +- id: mufu-1 + via: blackbox + drivers: + - simx + app: mufu +# Wide warp: 16 threads serialized onto 1 MUFU lane, 16 issue cycles per warp. +- id: mufu-nt16 + via: blackbox + drivers: + - simx + - rtlsim + app: mufu + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# One MUFU lane per SFU lane (non-default): nothing is serialized. +- id: mufu-full-lanes + via: blackbox + drivers: + - simx + - rtlsim + app: mufu + configs: -DVX_CFG_NUM_MUFU_LANES=4 +- id: mufu-xrt + via: blackbox + drivers: + - xrt + app: mufu + +# --------------------------------------------------------------------------- +# check: model_parity — SimX vs rtlsim (see core.yaml). Each run is sized to +# >=~300k cycles so steady-state MUFU throughput dominates. +# --------------------------------------------------------------------------- + +# Default shape: 4 threads over the 1 lane, repeated ex2 (~620k cycles). +- id: parity-mufu + check: model_parity + via: blackbox + app: mufu + args: -n1024 -r256 -tex2 +# 8 threads serialized onto the 1 lane, repeated sinpi (~547k cycles). +- id: parity-mufu-nt8 + check: model_parity + via: blackbox + app: mufu + args: -n2048 -r256 -tsinpi + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 +# 16 threads serialized onto the 1 lane, the fpga_gate shape (~532k cycles). +- id: parity-mufu-nt16 + check: model_parity + via: blackbox + app: mufu + args: -n2048 -r256 -tsinpi + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index 89c8ba08e8..be00b76514 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -38,3 +38,9 @@ tests: via: script run: "make -C hw/unittest/tex_lerp run" touches: [hw/rtl/tex, hw/unittest/tex_lerp, sw/common/vx_gfx_abi.h, sw/common/gfx_frag_tex.h] + # MUFU datapath (VX_mufu_pe) against the golden model SimX and the tests share + # (sw/common/mufu.cpp): specials, a strided sweep of all FP32 inputs, random. + - id: hw-mufu + via: script + run: "make -C hw/unittest/mufu_unit run" + touches: [hw/rtl/mufu, hw/unittest/mufu_unit, sw/common/mufu.cpp, sw/common/mufu_tables.h] diff --git a/ci/testcases/zfhmin.yaml b/ci/testcases/zfhmin.yaml new file mode 100644 index 0000000000..7e150cb385 --- /dev/null +++ b/ci/testcases/zfhmin.yaml @@ -0,0 +1,58 @@ +# Zfhmin: half-precision load/store, move and fp16<->fp32/fp64 conversion. +# The `zfhmin` app checks every conversion bit-exactly against softfloat, with +# fflags, across all rounding modes, exhaustively for the widening direction, +# plus flh/fsh/fmv NaN-boxing. The converter datapath is also covered by +# unittest:hw-fpu (fcvt_unit). +category: zfhmin +defaults: + xlen: + - 32 + - 64 + tier: smoke + touches: [hw/rtl/fpu, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_slice.sv, hw/rtl/core/VX_csr_unit.sv, hw/dpi/dpi_float.cpp, sim/simx/fpu_unit.cpp, sim/simx/decode.cpp, sim/simx/lsu_unit.cpp, sim/simx/csr_unit.cpp, sw/common/rvfloats.cpp] +tests: +- id: zfhmin-1 + via: blackbox + drivers: + - simx + - rtlsim + app: zfhmin + args: -n1024 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE +# The synthesizable converter (STD) rather than the DPI model rtlsim uses by default. +- id: zfhmin-std + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n256 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_FPU_TYPE_STD +- id: zfhmin-dsp + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n256 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_FPU_TYPE_DSP + xlen: + - 32 +# Wide warp with a single FPU lane: conversions split into one packet per thread. +- id: zfhmin-nt16-fpu1 + via: blackbox + drivers: + - rtlsim + app: zfhmin + args: -n0 -p0x23 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_FPU_LANES=1 + xlen: + - 32 + +# --------------------------------------------------------------------------- +# check: model_parity — SimX vs rtlsim on the narrowing/widening/boxing phases. +# --------------------------------------------------------------------------- +- id: parity-zfhmin + check: model_parity + via: blackbox + app: zfhmin + args: -n1024 -p0x23 + configs: -DVX_CFG_EXT_ZFHMIN_ENABLE diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index 7461d1677d..a4593709a3 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -214,6 +214,27 @@ A register **field** may also carry an immediate. The tensor instructions put formats and flags in `rd`, `rs1` and `rs2`; decode reads the field without reading a register. +### Opcode map + +Allocated custom opcode space. Check here before claiming a slot, and update +the table in the same change. + +| Opcode | Split by | Value | Extension | Format | +|---|---|---|---|---| +| custom-0 `0x0B` | `funct7` | `0x00` | warp control (`tmc`, `wspawn`, `split`, `join`, `bar`, `pred`, `wsync`) | R | +| | | `0x01` | vote / shuffle | R | +| | | `0x02` | tensor core (WMMA/WGMMA) | R | +| | | `0x03` | DXA async copy | R | +| | | `0x04` | packed loads (`vx_packl*`) | R | +| custom-1 `0x2B` | `funct3` | `0` | `vx_wgather` | R4 | +| | | `3` | `vx_om_export` | R4 | +| | | `5` | `vx_tex` | R4 | +| | | `6` | RTU hit window / callback return | R4 | +| | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | +| | | `1`, `2`, `4` | free | | +| custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | +| custom-3 `0x7B` | | | free | | + --- ## 4. How a wide-operand instruction executes diff --git a/docs/designs/floating_point_unit.md b/docs/designs/floating_point_unit.md index 65cd877c8a..7e9e9d4eb0 100644 --- a/docs/designs/floating_point_unit.md +++ b/docs/designs/floating_point_unit.md @@ -85,11 +85,14 @@ Decode emits a 4-bit operation, a 2-bit `fmt`, and a 3-bit `frm` | `1000` / `1001` | `F2I` / `F2U` | CVT | `[0]` float is double, `[1]` integer is 64-bit | rounding mode | | `1010` / `1011` | `I2F` / `U2F` | CVT | `[0]` float is double, `[1]` integer is 64-bit | rounding mode | | `1100` | `CMP` | NCP | `[0]` double | `LE` = 0, `LT` = 1, `EQ` = 2 | -| `1101` | `F2F` | **CVT** | `[0]` destination is double | rounding mode | -| `1110` | `MISC` | NCP | `[0]` double | `SGNJ`, `SGNJN`, `SGNJX`, `CLASS`, `MVXW`, `MVWX`, `FMIN`, `FMAX` = 0–7 | +| `1101` | `F2F` | **CVT** | `[0]` destination is double, `[1]` source is half (Zfhmin) | rounding mode | +| `1110` | `MISC` | NCP | `[0]` double, `[1]` half for `MVXW`/`MVWX` (Zfhmin) | `SGNJ`, `SGNJN`, `SGNJX`, `CLASS`, `MVXW`, `MVWX`, `FMIN`, `FMAX` = 0–7 | +| `1111` | `F2H` | **CVT** | `[0]` source is double | rounding mode | -`F2F` is the one exception to the index rule: its top bits say NCP, and the -request switch steers it to CVT. +`F2F` and `F2H` are the exceptions to the index rule: their top bits say NCP, +and the request switch steers them to CVT. `F2H` (`FCVT.H.S`, `FCVT.H.D`) and +the half forms of `F2F` and the moves exist only with +`VX_CFG_EXT_ZFHMIN_ENABLE`. Subtraction is not an operation. `SUB`, `MSUB` and `NMSUB` are `ADD`, `MADD` and `NMADD` with `fmt[1]` set, and the FMA core folds the sign into its @@ -291,9 +294,15 @@ value sized to the widest enabled format. - **Unpack** selects the source format's fields, re-biases, and left-aligns. - **Round and pack** narrows the canonical value to the destination through one shared tree, reused for every format pair. -- **Narrowing** (`FCVT.S.D`) handles a subnormal destination by - denormalizing and rounding with a correct guard/round/sticky split, and - overflow to infinity. +- **Narrowing** (`FCVT.S.D`, and `FCVT.H.S`/`FCVT.H.D` with Zfhmin) handles a + subnormal destination by denormalizing and rounding with a correct + guard/round/sticky split; a subnormal that rounds up becomes the smallest + normal. Underflow uses RISC-V's after-rounding tininess. Overflow yields + infinity or the largest finite value according to the rounding mode, and a + rounding carry into the infinity exponent also raises OF. +- **Half** (`HALF_ENABLE`, set by `VX_CFG_EXT_ZFHMIN_ENABLE`) adds an F16 + source and destination; a half source that is not NaN-boxed reads as the + canonical NaN. - The integer side is `XLEN` wide: I32, I64, U32, U64. Latency is `VX_CFG_FCVT_LATENCY` on a 32-bit build and **one more on a @@ -431,7 +440,8 @@ flight, and a result waiting on the commit arbiter keeps its tag: | what | where | checks | |---|---|---| -| leaf units | `hw/unittest/{fma,fdiv,fsqrt,fcvt,fdivsqrt}_unit`, run by `make -C hw/unittest run-fpu` | bit-exact result and flags against soft-float, over every RISC-V rounding mode, in the default and the area-reduced configurations | +| leaf units | `hw/unittest/{fma,fdiv,fsqrt,fcvt,fdivsqrt}_unit`, run by `make -C hw/unittest run-fpu` | bit-exact result and flags against soft-float, over every RISC-V rounding mode, in the default and the area-reduced configurations (`fcvt` includes F32↔F16) | +| Zfhmin | `zfhmin` category: the `zfhmin` app on simx and rtlsim, with the DPI, STD and DSP backends | every half conversion bit-exact with flags across all rounding modes (widening exhaustive), `flh`/`fsh`/`fmv` NaN-boxing | | FPU RTL end to end | `riscv` category, `isa-32f-dsp`: `rtlsim` rebuilt with the DSP backend, running `rv32uf` | the soft cores under the full pipeline, F32 | | ISA conformance | `riscv` category, every other lane: `rv32uf`, `rv64uf`, `rv64ud` | decode, the front end and writeback — arithmetic comes from the DPI model | | STD backend on hardware | `config2` category, `dogfood` with `-DVX_CFG_FPU_TYPE_STD` on `xrt` | the soft cores on the FPGA | @@ -444,6 +454,15 @@ divergence from soft-float fails CI. exercise the FPU's arithmetic RTL. `isa-32f-dsp` is the one lane that does, and it is F32 only: the F64 soft cores are covered by the unit tests alone. +### Special functions are not in the FPU + +Approximate transcendentals (`rcp`, `rsqrt`, `ex2`, `lg2`, `sinpi`, `cospi`, +`tanh`) run on the MUFU, a processing element of the SFU, not on an FPU core. +The SFU's lanes are serialized onto `VX_CFG_NUM_MUFU_LANES` datapath lanes +(default 1), so FPU and MUFU instructions from different warps execute +concurrently; it ignores `frm` and never writes `fflags`. The FPU's +`fsqrt`/`fdiv` remain the correctly rounded operations. + --- ## 13. Known limitations @@ -460,3 +479,8 @@ and it is F32 only: the F64 soft cores are covered by the unit tests alone. models `VX_CFG_FCVT_LATENCY` on every build, one cycle short. - **Double precision on a 32-bit build** (§4.1) is not supported by the operand path. +- **FPNEW narrowing of infinity.** The pinned cvfpu treats an infinite F2F + source as an overflow, so `RTZ`, and `RDN`/`RUP` toward zero, return the + largest finite value instead of infinity (`FCVT.S.D`, `FCVT.H.S`, + `FCVT.H.D`), and `FCVT.H.S` of a signaling NaN does not raise NV. The DPI, + STD and DSP backends are correct. diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 6436335429..0ba4a8bdb5 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -2,7 +2,7 @@ ## Overview -The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 75 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. +The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 76 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. All modules are parameterized for data width, depth, and implementation strategy, with support for both FPGA (Xilinx/Intel) and ASIC synthesis flows. @@ -1260,6 +1260,25 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn --- +### VX_rom + +**File:** `VX_rom.sv` +**Purpose:** Read-only memory whose contents are a packed parameter, so it needs no init file. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DATAW` | 1 | Data width in bits | +| `SIZE` | 1 | Number of entries | +| `OUT_REG` | 0 | Registered read (one cycle) instead of an asynchronous read | +| `INIT_DATA` | `'0` | Contents, `SIZE*DATAW` bits; entry `i` is `INIT_DATA[i*DATAW +: DATAW]` | + +**Features:** +- Contents are elaborated as constants, so they survive the ASIC flow, where `VX_sp_ram`'s `INIT_FILE` loading is compiled out +- Under synthesis, a registered ROM that passes `FORCE_BRAM` is tagged for block ROM (`rom_style`/`romstyle`); otherwise it maps to logic +- Generated tables (e.g. `hw/rtl/mufu/VX_mufu_rom.sv`) instantiate it directly + +--- + ### VX_async_ram_patch **File:** `VX_async_ram_patch.sv` @@ -1858,27 +1877,28 @@ Infrastructure for runtime debug observation and serial trace extraction. | 49 | VX_priority_encoder | Encoder | Small | O(log N) | | 50 | VX_reduce_tree | Arithmetic | Small | O(log N) | | 51 | VX_reset_relay | Sync | Small | 1 cycle | -| 52 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | -| 53 | VX_scan | Encoder | Medium | O(log N) | -| 54 | VX_scope_switch | Debug | Small | 0 cycles | -| 55 | VX_scope_tap | Debug | Large | Multi-cycle | -| 56 | VX_serial_div | Arithmetic | Small | Multi-cycle | -| 57 | VX_serial_mul | Arithmetic | Small | Multi-cycle | -| 58 | VX_shift_register | Buffer | Varies | N cycles | -| 59 | VX_skid_buffer | Buffer | Small | 0-1 cycles | -| 60 | VX_sp_ram | Memory | Varies | 0-1 cycles | -| 61 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | -| 62 | VX_stream_buffer | Buffer | Small | 0-1 cycles | -| 63 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | -| 64 | VX_stream_fork | Stream Util | Small | 0 cycles | -| 65 | VX_stream_join | Stream Util | Small | 0 cycles | -| 66 | VX_stream_omega | Interconnect | Large | O(log N) | -| 67 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | -| 68 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | -| 69 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | -| 70 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | -| 71 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | -| 72 | VX_ticket_lock | Allocator | Small | 0-1 cycles | -| 73 | VX_toggle_buffer | Buffer | Small | 1 cycle | -| 74 | VX_transpose | Bit Manip | Zero | 0 cycles | -| 75 | VX_wallace_mul | Arithmetic | Large | O(log N) | +| 52 | VX_rom | Memory | Varies | 0-1 cycles | +| 53 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | +| 54 | VX_scan | Encoder | Medium | O(log N) | +| 55 | VX_scope_switch | Debug | Small | 0 cycles | +| 56 | VX_scope_tap | Debug | Large | Multi-cycle | +| 57 | VX_serial_div | Arithmetic | Small | Multi-cycle | +| 58 | VX_serial_mul | Arithmetic | Small | Multi-cycle | +| 59 | VX_shift_register | Buffer | Varies | N cycles | +| 60 | VX_skid_buffer | Buffer | Small | 0-1 cycles | +| 61 | VX_sp_ram | Memory | Varies | 0-1 cycles | +| 62 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | +| 63 | VX_stream_buffer | Buffer | Small | 0-1 cycles | +| 64 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | +| 65 | VX_stream_fork | Stream Util | Small | 0 cycles | +| 66 | VX_stream_join | Stream Util | Small | 0 cycles | +| 67 | VX_stream_omega | Interconnect | Large | O(log N) | +| 68 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | +| 69 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | +| 70 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | +| 71 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | +| 72 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | +| 73 | VX_ticket_lock | Allocator | Small | 0-1 cycles | +| 74 | VX_toggle_buffer | Buffer | Small | 1 cycle | +| 75 | VX_transpose | Bit Manip | Zero | 0 cycles | +| 76 | VX_wallace_mul | Arithmetic | Large | O(log N) | diff --git a/hw/dpi/dpi_float.cpp b/hw/dpi/dpi_float.cpp index b29dfe51fc..5e59e6559b 100644 --- a/hw/dpi/dpi_float.cpp +++ b/hw/dpi/dpi_float.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "svdpi.h" #include "verilated_vpi.h" @@ -51,117 +52,48 @@ extern "C" { void dpi_fmax(bool enable, int dst_fmt, int64_t a, int64_t b, int64_t* result, svBitVecVal* fflags); } +constexpr uint32_t kRegBits = VX_CFG_XLEN; + inline uint64_t nan_box(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffff00000000; -#else - return value; -#endif + return nanbox::box(value, 32, kRegBits); } inline uint64_t nan_box16(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffffffff0000; -#else - return value; -#endif + return nanbox::box(value, 16, kRegBits); } inline uint64_t nan_box19(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xfffffffffff80000; -#else - return value; -#endif + return nanbox::box(value, 19, kRegBits); } inline uint64_t nan_box8(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xffffffffffffff00; -#else - return value; -#endif + return nanbox::box(value, 8, kRegBits); } inline uint64_t nan_box4(uint32_t value) { -#ifdef VX_CFG_XLEN_64 - return value | 0xfffffffffffffff0; -#else - return value; -#endif -} - -inline bool is_nan_boxed(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint32_t(value >> 32) == 0xffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed16(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 16) == 0xffffffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed19(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 19) == 0xffffffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed8(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 8) == 0xffffffffffffff); -#else - return true; -#endif -} - -inline bool is_nan_boxed4(uint64_t value) { -#ifdef VX_CFG_XLEN_64 - return (uint64_t(value >> 4) == 0xfffffffffffffff); -#else - return true; -#endif + return nanbox::box(value, 4, kRegBits); } inline int64_t check_boxing(int64_t a) { - if (is_nan_boxed(a)) - return a; - return nan_box(0x7fc00000); // NaN + return nanbox::is_boxed(a, 32, kRegBits) ? a : nan_box(0x7fc00000); } inline int64_t check_boxing16(int64_t a, uint16_t nan_value) { - if (is_nan_boxed16(a)) - return a; - return nan_box16(nan_value); // NaN + return nanbox::is_boxed(a, 16, kRegBits) ? a : nan_box16(nan_value); } inline int64_t check_boxing19(int64_t a, uint32_t nan_value) { - if (is_nan_boxed19(a)) - return a; - return nan_box19(nan_value); // NaN + return nanbox::is_boxed(a, 19, kRegBits) ? a : nan_box19(nan_value); } inline int64_t check_boxing8(int64_t a, uint8_t nan_value) { - if (is_nan_boxed8(a)) - return a; - return nan_box8(nan_value); // NaN + return nanbox::is_boxed(a, 8, kRegBits) ? a : nan_box8(nan_value); } inline int64_t check_boxing4(int64_t a, uint8_t nan_value) { - if (is_nan_boxed4(a)) - return a; - return nan_box4(nan_value); // NaN + return nanbox::is_boxed(a, 4, kRegBits) ? a : nan_box4(nan_value); } - void dpi_fadd(bool enable, int dst_fmt, int64_t a, int64_t b, const svBitVecVal* frm, int64_t* result, svBitVecVal* fflags) { if (!enable) return; @@ -360,6 +292,9 @@ void dpi_f2f(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal case 0: // fp32 *result = rv_ftod(check_boxing(a), *frm, fflags); break; + case 2: // fp16 + *result = rv_htod(check_boxing16(a, 0x7E00), *frm, fflags); + break; default: std::abort(); } @@ -369,6 +304,9 @@ void dpi_f2f(bool enable, int dst_fmt, int src_fmt, int64_t a, const svBitVecVal case 0: // fp32 *result = nan_box16(rv_ftoh_s(check_boxing(a), *frm, fflags)); break; + case 1: // fp64 + *result = nan_box16(rv_dtoh(a, *frm, fflags)); + break; default: std::abort(); } diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index b2bf9f07c2..7607b2bdf6 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -95,6 +95,18 @@ `endif `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + `ifndef VX_CFG_EXT_F_ENABLE + `error "VX_CFG_EXT_MUFU_ENABLE requires VX_CFG_EXT_F_ENABLE" + `endif +`endif + +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + `ifndef VX_CFG_EXT_F_ENABLE + `error "VX_CFG_EXT_ZFHMIN_ENABLE requires VX_CFG_EXT_F_ENABLE" + `endif +`endif + // Convenience flag: the TCU metadata SRAM is present when any metadata-consuming // mode (MX or sparse) is enabled. Internal derived macro — not a VX_CFG_* knob. `ifdef VX_CFG_TCU_MX_ENABLE diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index d836c5a760..53193421c1 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -489,6 +489,7 @@ package VX_gpu_pkg; localparam INST_FPU_CMP = 4'b1100; // frm: LE=0, LT=1, EQ=2 localparam INST_FPU_F2F = 4'b1101; // fmt[0]: F32=0, F64=1 localparam INST_FPU_MISC = 4'b1110; // frm: SGNJ=0, SGNJN=1, SGNJX=2, CLASS=3, MVXW=4, MVWX=5, FMIN=6, FMAX=7 + localparam INST_FPU_F2H = 4'b1111; localparam INST_FPU_BITS = 4; function automatic logic inst_fpu_is_class(input logic [INST_FPU_BITS-1:0] op, input logic [INST_FRM_BITS-1:0] frm); @@ -525,6 +526,9 @@ package VX_gpu_pkg; `endif `ifdef EXT_GFX_ANY_ENABLE localparam INST_SFU_RTUW = 4'hE; // RTU hit window (GETW/GETWF) + trace ops +`endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam INST_SFU_MUFU = 4'hF; `endif localparam INST_SFU_BITS = 4; @@ -546,6 +550,17 @@ package VX_gpu_pkg; return (op >= 6 && op <= 8); endfunction +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam INST_MUFU_RCP = 3'h0; + localparam INST_MUFU_RSQRT = 3'h1; + localparam INST_MUFU_EX2 = 3'h2; + localparam INST_MUFU_LG2 = 3'h3; + localparam INST_MUFU_SINPI = 3'h4; + localparam INST_MUFU_COSPI = 3'h5; + localparam INST_MUFU_TANH = 3'h6; + localparam INST_MUFU_BITS = 3; +`endif + /////////////////////////////// Issue parameters ////////////////////////// localparam ISSUE_ISW_BITS = `CLOG2(`VX_CFG_ISSUE_WIDTH); @@ -959,6 +974,14 @@ package VX_gpu_pkg; `PACKAGE_ASSERT($bits(tcu_args_t) == INST_ARGS_BITS) `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + typedef struct packed { + logic [INST_ARGS_BITS-INST_MUFU_BITS-1:0] __padding; + logic [INST_MUFU_BITS-1:0] fn; + } mufu_args_t; + `PACKAGE_ASSERT($bits(mufu_args_t) == INST_ARGS_BITS) +`endif + `ifdef VX_CFG_EXT_TEX_ENABLE // Texture sample op args (vx_tex, CUSTOM1 funct3=5, R4-type): u/v/lod ride // rs1/rs2/rs3, the texel lands in rd, and `stage` (funct2) selects the @@ -1018,6 +1041,9 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_TCU_ENABLE tcu_args_t tcu; `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + mufu_args_t mufu; + `endif `ifdef VX_CFG_EXT_TEX_ENABLE tex_args_t tex; `endif diff --git a/hw/rtl/VX_platform.vh b/hw/rtl/VX_platform.vh index 9b4ddd636d..85fae1d635 100644 --- a/hw/rtl/VX_platform.vh +++ b/hw/rtl/VX_platform.vh @@ -170,6 +170,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM (* ramstyle = "block" *) +`define USE_BLOCK_ROM (* romstyle = "M20K" *) `define USE_FAST_BRAM (* ramstyle = "MLAB, no_rw_check" *) `define NO_RW_RAM_CHECK (* altera_attribute = "-name add_pass_through_logic_to_inferred_rams off" *) `define RW_RAM_CHECK (* altera_attribute = "-name add_pass_through_logic_to_inferred_rams on" *) @@ -183,6 +184,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM (* ram_style = "block" *) +`define USE_BLOCK_ROM (* rom_style = "block" *) `define USE_FAST_BRAM (* ram_style = "distributed" *) `define NO_RW_RAM_CHECK (* rw_addr_collision = "no" *) `define RW_RAM_CHECK (* rw_addr_collision = "yes" *) @@ -199,6 +201,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) 0 `define USE_BLOCK_BRAM +`define USE_BLOCK_ROM `define USE_FAST_BRAM `define NO_RW_RAM_CHECK `define RW_RAM_CHECK @@ -212,6 +215,7 @@ `define FORCE_BRAM(d,w) (((d) >= 64) && (((d) * (w)) >= 2048)) `define FORCE_BUILTIN_ADDER(w) ((w) <= 27) `define USE_BLOCK_BRAM +`define USE_BLOCK_ROM `define USE_FAST_BRAM `define NO_RW_RAM_CHECK `define RW_RAM_CHECK diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index dd16cd3dc7..8c3f8933c1 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -269,6 +269,20 @@ package VX_trace_pkg; `endif `ifdef VX_CFG_EXT_RASTER_ENABLE INST_SFU_RASTER: `TRACE(level, ("RAST.FETCH")) + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + INST_SFU_MUFU: begin + case (op_args.mufu.fn) + INST_MUFU_RCP: `TRACE(level, ("MUFU.RCP")) + INST_MUFU_RSQRT: `TRACE(level, ("MUFU.RSQRT")) + INST_MUFU_EX2: `TRACE(level, ("MUFU.EX2")) + INST_MUFU_LG2: `TRACE(level, ("MUFU.LG2")) + INST_MUFU_SINPI: `TRACE(level, ("MUFU.SINPI")) + INST_MUFU_COSPI: `TRACE(level, ("MUFU.COSPI")) + INST_MUFU_TANH: `TRACE(level, ("MUFU.TANH")) + default: `TRACE(level, ("MUFU.?")) + endcase + end `endif default: `TRACE(level, ("?")) endcase @@ -360,10 +374,18 @@ package VX_trace_pkg; end end INST_FPU_F2F: begin + case (op_args.fpu.fmt) + 2'b00: `TRACE(level, ("FCVT.S.D")) + 2'b01: `TRACE(level, ("FCVT.D.S")) + 2'b10: `TRACE(level, ("FCVT.S.H")) + 2'b11: `TRACE(level, ("FCVT.D.H")) + endcase + end + INST_FPU_F2H: begin if (op_args.fpu.fmt[0]) begin - `TRACE(level, ("FCVT.D.S")) + `TRACE(level, ("FCVT.H.D")) end else begin - `TRACE(level, ("FCVT.S.D")) + `TRACE(level, ("FCVT.H.S")) end end INST_FPU_F2I: begin @@ -416,8 +438,8 @@ package VX_trace_pkg; 1: `TRACE(level, ("FSGNJN.S")) 2: `TRACE(level, ("FSGNJX.S")) 3: `TRACE(level, ("FCLASS.S")) - 4: `TRACE(level, ("FMV.X.S")) - 5: `TRACE(level, ("FMV.S.X")) + 4: `TRACE(level, (op_args.fpu.fmt[1] ? "FMV.X.H" : "FMV.X.S")) + 5: `TRACE(level, (op_args.fpu.fmt[1] ? "FMV.H.X" : "FMV.S.X")) 6: `TRACE(level, ("FMIN.S")) 7: `TRACE(level, ("FMAX.S")) endcase diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index 925d1e7cff..ae0b7038b2 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -68,12 +68,28 @@ module VX_csr_unit import VX_gpu_pkg::*; #( assign execute_if.ready = csr_req_ready && cta_read_done; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rs1_data; - `UNUSED_VAR (rs1_data) for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rs1_data assign rs1_data[i] = execute_if.data.rs1_data[i]; end - wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_data[0]; + wire [`VX_CFG_XLEN-1:0] rs1_first; + if (NUM_LANES > 1) begin : g_rs1_first + wire [`LOG2UP(NUM_LANES)-1:0] first_lane; + VX_priority_encoder #( + .N (NUM_LANES), + .REVERSE (0) + ) first_lane_sel ( + .data_in (execute_if.data.header.tmask), + .index_out (first_lane), + `UNUSED_PIN (onehot_out), + `UNUSED_PIN (valid_out) + ); + assign rs1_first = rs1_data[first_lane]; + end else begin : g_rs1_first_0 + assign rs1_first = rs1_data[0]; + end + + wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_first; wire [`VX_CFG_XLEN-1:0] csr_scalar_data, dcr_read_data; diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 9c2a3fa44d..12a59ca372 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -61,6 +61,15 @@ module VX_decode import wire [4:0] rs3 = instr[31:27]; `UNUSED_VAR (funct2) + +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire fcvt_fmt_valid = (rs2[4:2] == 3'b000) && (rs2[1:0] != 2'b11) && (funct2 != 2'b11) && (rs2[1:0] != funct2); +`ifdef VX_CFG_FLEN_64 + wire fcvt_fmt_supported = fcvt_fmt_valid; +`else + wire fcvt_fmt_supported = fcvt_fmt_valid && (rs2[1:0] != 2'b01) && (funct2 != 2'b01); +`endif +`endif `ifndef VX_CFG_EXT_A_ENABLE `UNUSED_VAR (funct5) `endif @@ -506,7 +515,21 @@ module VX_decode import `USED_FREG (rs1); `USED_FREG (rs2); end - `ifdef VX_CFG_FLEN_64 + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + 5'b01000: begin + if (fcvt_fmt_supported) begin + if (funct2 == 2'b10) begin + op_type = INST_OP_BITS'(INST_FPU_F2H); + op_args.fpu.fmt[0] = rs2[0]; + end else begin + op_type = INST_OP_BITS'(INST_FPU_F2F); + end + rd_xregs[XREG_1] = frm_is_dyn; + `USED_FREG (rd); + `USED_FREG (rs1); + end + end + `elsif VX_CFG_FLEN_64 5'b01000: begin // FCVT.S.D, FCVT.D.S op_type = INST_OP_BITS'(INST_FPU_F2F); @@ -560,6 +583,9 @@ module VX_decode import // NCP: FMV.X.W=4 op_type = INST_OP_BITS'(INST_FPU_MISC); op_args.fpu.frm = INST_FRM_BITS'(4); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + op_args.fpu.fmt[1] = funct2[1]; + `endif end wr_xregs[XREG_0] = 1'b0; `USED_IREG (rd); @@ -569,6 +595,9 @@ module VX_decode import // NCP: FMV.W.X=5 op_type = INST_OP_BITS'(INST_FPU_MISC); op_args.fpu.frm = INST_FRM_BITS'(5); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + op_args.fpu.fmt[1] = funct2[1]; + `endif wr_xregs[XREG_0] = 1'b0; `USED_FREG (rd); `USED_IREG (rs1); @@ -880,6 +909,18 @@ module VX_decode import default:; endcase end + `ifdef VX_CFG_EXT_MUFU_ENABLE + INST_EXT3: begin + if (funct7 == 7'h00 && funct3 == 3'h0 && rs2 <= 5'(INST_MUFU_TANH)) begin + ex_type = EX_SFU; + op_type = INST_OP_BITS'(INST_SFU_MUFU); + op_args.mufu.__padding = '0; + op_args.mufu.fn = INST_MUFU_BITS'(rs2); + `USED_FREG (rd); + `USED_FREG (rs1); + end + end + `endif default:; endcase end diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index affb932298..5a54698dd6 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -413,13 +413,16 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] rsp_data; -`ifdef VX_CFG_XLEN_64 `ifdef VX_CFG_EXT_F_ENABLE // apply nan-boxing to flw outputs wire rsp_is_float = (get_reg_type(rsp_hdr.rd) == REG_TYPE_F); `else wire rsp_is_float = 0; `endif +`ifndef VX_CFG_XLEN_64 +`ifndef VX_CFG_EXT_ZFHMIN_ENABLE + `UNUSED_VAR (rsp_is_float) +`endif `endif for (genvar i = 0; i < NUM_LANES; i++) begin : g_rsp_data @@ -435,7 +438,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( always @(*) begin case (inst_lsu_fmt(rsp_op_type)) LSU_FMT_B: rsp_data[i] = `VX_CFG_XLEN'(signed'(rsp_data8)); + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + LSU_FMT_H: rsp_data[i] = rsp_is_float ? {{(`VX_CFG_XLEN-16){1'b1}}, rsp_data16} : `VX_CFG_XLEN'(signed'(rsp_data16)); + `else LSU_FMT_H: rsp_data[i] = `VX_CFG_XLEN'(signed'(rsp_data16)); + `endif LSU_FMT_BU: rsp_data[i] = `VX_CFG_XLEN'(unsigned'(rsp_data8)); LSU_FMT_HU: rsp_data[i] = `VX_CFG_XLEN'(unsigned'(rsp_data16)); `ifdef VX_CFG_XLEN_64 diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index 660303ee31..d7a200d11a 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -68,7 +68,7 @@ import VX_raster_pkg::*; `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = 1; localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; - localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; + localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_MUFU_ENABLED; localparam PE_SEL_BITS = `CLOG2(PE_COUNT); localparam PE_IDX_WCTL = 0; localparam PE_IDX_CSRS = 1; @@ -87,6 +87,9 @@ import VX_raster_pkg::*; `ifdef VX_CFG_EXT_RTU_ENABLE localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam PE_IDX_MUFU = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; +`endif VX_execute_if #( .data_t (sfu_execute_t) @@ -146,6 +149,11 @@ import VX_raster_pkg::*; pe_select = PE_SEL_BITS'(PE_IDX_RTUW); end `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + if (per_block_execute_if[0].data.op_type == INST_SFU_MUFU) begin + pe_select = PE_SEL_BITS'(PE_IDX_MUFU); + end + `endif end VX_pe_switch #( @@ -291,6 +299,18 @@ import VX_raster_pkg::*; ); `endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + VX_mufu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) mufu_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MUFU]), + .result_if (pe_result_if[PE_IDX_MUFU]) + ); +`endif + VX_lane_gather #( .BLOCK_SIZE (BLOCK_SIZE), .NUM_LANES (NUM_LANES), diff --git a/hw/rtl/fpu/VX_fcvt_unit.sv b/hw/rtl/fpu/VX_fcvt_unit.sv index 8cbeeed93e..c14bdbcc4e 100644 --- a/hw/rtl/fpu/VX_fcvt_unit.sv +++ b/hw/rtl/fpu/VX_fcvt_unit.sv @@ -28,7 +28,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // sources to signed zero) for area. Use 0 for relaxed paths. parameter SUBNORM_ENABLE = 1, // 1: produce fflags (NV/OF/UF/NX). 0: tie fflags to 0 (area). - parameter EXCEPT_ENABLE = 1 + parameter EXCEPT_ENABLE = 1, + parameter HALF_ENABLE = 0 ) ( input wire clk, input wire reset, @@ -45,6 +46,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; input wire is_int64, // integer operand width: 1=I64, 0=I32 input wire src_fmt, // source float format: 1=F64, 0=F32 input wire dst_fmt, // dest float format: 1=F64, 0=F32 + input wire src_h, + input wire dst_h, input wire [`VX_CFG_XLEN-1:0] dataa, output wire [`VX_CFG_XLEN-1:0] result, @@ -53,7 +56,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; ); localparam F32_EXP = 8, F32_MAN = 23, F32_BIAS = 127; localparam F64_EXP = 11, F64_MAN = 52, F64_BIAS = 1023; + localparam F16_EXP = 5, F16_MAN = 10, F16_BIAS = 15; localparam HAS_D = (FLEN >= 64); + localparam HAS_H = (HALF_ENABLE != 0); // Widest enabled float format drives the canonical internal datapath. localparam SUPER_MAN = HAS_D ? F64_MAN : F32_MAN; @@ -67,9 +72,13 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; `UNUSED_VAR (src_fmt) `UNUSED_VAR (dst_fmt) `UNUSED_VAR (is_f2f) - wire src_is_d = HAS_D & src_fmt; - wire dst_is_d = HAS_D & dst_fmt; - wire f2f = HAS_D & is_f2f; + `UNUSED_VAR (src_h) + `UNUSED_VAR (dst_h) + wire src_is_h = HAS_H & src_h; + wire dst_is_h = HAS_H & dst_h; + wire src_is_d = HAS_D & src_fmt & ~src_is_h; + wire dst_is_d = HAS_D & dst_fmt & ~dst_is_h; + wire f2f = (HAS_D | HAS_H) & is_f2f; reg [LATENCY-1:0] mask_pipe; always @(posedge clk) begin @@ -88,27 +97,59 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [63:0] safe_dataa = 64'(dataa); + wire [15:0] h_op = (&dataa[`VX_CFG_XLEN-1:16]) ? dataa[15:0] : 16'h7E00; + + wire [31:0] s_op; + if (HAS_D) begin : g_s_unbox + assign s_op = (&dataa[`VX_CFG_XLEN-1:32]) ? dataa[31:0] : 32'h7FC00000; + end else begin : g_s_raw + assign s_op = dataa[31:0]; + end + // Float source field extraction (per source format) - wire input_fp_sgn = src_is_d ? safe_dataa[63] : safe_dataa[31]; + wire input_fp_sgn = src_is_h ? h_op[15] : src_is_d ? safe_dataa[63] : s_op[31]; fclass_t fclass32; - VX_fp_classifier #(.EXP_BITS(F32_EXP), .MAN_BITS(F32_MAN)) fp_classifier32 ( - .exp_i (safe_dataa[F32_MAN +: F32_EXP]), - .man_i (safe_dataa[F32_MAN-1:0]), + VX_fp_classifier #( + .EXP_BITS (F32_EXP), + .MAN_BITS (F32_MAN) + ) fp_classifier32 ( + .exp_i (s_op[F32_MAN +: F32_EXP]), + .man_i (s_op[F32_MAN-1:0]), .clss_o (fclass32) ); - fclass_t fclass; + fclass_t fclass_sd; if (HAS_D) begin : g_classify_d fclass_t fclass64; - VX_fp_classifier #(.EXP_BITS(F64_EXP), .MAN_BITS(F64_MAN)) fp_classifier64 ( + VX_fp_classifier #( + .EXP_BITS (F64_EXP), + .MAN_BITS (F64_MAN) + ) fp_classifier64 ( .exp_i (safe_dataa[F64_MAN +: F64_EXP]), .man_i (safe_dataa[F64_MAN-1:0]), .clss_o (fclass64) ); - assign fclass = src_is_d ? fclass64 : fclass32; + assign fclass_sd = src_is_d ? fclass64 : fclass32; end else begin : g_classify_s - assign fclass = fclass32; + assign fclass_sd = fclass32; + end + + fclass_t fclass; + if (HAS_H) begin : g_classify_h + fclass_t fclass16; + VX_fp_classifier #( + .EXP_BITS (F16_EXP), + .MAN_BITS (F16_MAN) + ) fp_classifier16 ( + .exp_i (h_op[F16_MAN +: F16_EXP]), + .man_i (h_op[F16_MAN-1:0]), + .clss_o (fclass16) + ); + assign fclass = src_is_h ? fclass16 : fclass_sd; + end else begin : g_classify_no_h + `UNUSED_VAR (h_op) + assign fclass = fclass_sd; end // Source significand {hidden, mantissa} placed in the low bits, and the @@ -117,17 +158,20 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // signed zero (mantissa forced 0 -> downstream zero path). wire src_daz = (SUBNORM_ENABLE == 0) & fclass.is_subnormal; wire [S_MAN_WIDTH-1:0] fp_unpacked_mant = src_daz ? '0 + : src_is_h ? S_MAN_WIDTH'({fclass.is_normal, h_op[F16_MAN-1:0]}) : src_is_d ? S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F64_MAN-1:0]}) - : S_MAN_WIDTH'({fclass.is_normal, safe_dataa[F32_MAN-1:0]}); + : S_MAN_WIDTH'({fclass.is_normal, s_op[F32_MAN-1:0]}); - wire signed [S_EXP_WIDTH-1:0] src_bias_s = src_is_d ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); - wire [SUPER_EXP-1:0] src_exp_raw; + wire signed [S_EXP_WIDTH-1:0] src_bias_s = src_is_h ? S_EXP_WIDTH'(F16_BIAS) + : src_is_d ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); + wire [SUPER_EXP-1:0] src_exp_sd; if (HAS_D) begin : g_srcexp_d - assign src_exp_raw = src_is_d ? safe_dataa[F64_MAN +: F64_EXP] - : SUPER_EXP'(safe_dataa[F32_MAN +: F32_EXP]); + assign src_exp_sd = src_is_d ? safe_dataa[F64_MAN +: F64_EXP] + : SUPER_EXP'(s_op[F32_MAN +: F32_EXP]); end else begin : g_srcexp_s - assign src_exp_raw = safe_dataa[F32_MAN +: F32_EXP]; + assign src_exp_sd = s_op[F32_MAN +: F32_EXP]; end + wire [SUPER_EXP-1:0] src_exp_raw = src_is_h ? SUPER_EXP'(h_op[F16_MAN +: F16_EXP]) : src_exp_sd; wire i_sign = safe_dataa[is_int64 ? (`VX_CFG_XLEN-1) : 31] && is_signed; wire [`VX_CFG_XLEN-1:0] i_mag_raw = i_sign ? (-dataa) : dataa; @@ -149,21 +193,21 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; end // Stage 0 -> Stage 1 - wire is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, src_d_s1, dst_d_s1; + wire is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1; wire [INST_FRM_BITS-1:0] frm_s1; fclass_t fclass_s1; wire [S_MAN_WIDTH-1:0] unpacked_mant_s1; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s1; VX_pipe_register #( - .DATAW (8 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH), + .DATAW (10 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH), .DEPTH (LATENCY > 5) ) pipe_reg1 ( .clk (clk), .reset (reset), .enable (enable && mask), - .data_in ({is_itof, is_ftoi, f2f, is_signed, is_int64, src_is_d, dst_is_d, input_sign_s0, frm, fclass, unpacked_mant_s0, unpacked_exp_s0}), - .data_out ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, src_d_s1, dst_d_s1, input_sign_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1}) + .data_in ({is_itof, is_ftoi, f2f, is_signed, is_int64, src_is_d, dst_is_d, src_is_h, dst_is_h, input_sign_s0, frm, fclass, unpacked_mant_s0, unpacked_exp_s0}), + .data_out ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1, input_sign_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1}) ); // ====================================================================== @@ -226,7 +270,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire mant_is_zero_s1 = ~mant_is_nonzero_s1; // Stage 1 -> Stage 2 - wire is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2; + wire is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, src_h_s2, dst_h_s2; wire [INST_FRM_BITS-1:0] frm_s2; fclass_t fclass_s2; wire [S_MAN_WIDTH-1:0] unpacked_mant_s2; @@ -234,14 +278,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [LZC_RESULT_WIDTH-1:0] renorm_shamt_s2; VX_pipe_register #( - .DATAW (9 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + LZC_RESULT_WIDTH), + .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + LZC_RESULT_WIDTH), .DEPTH (LATENCY > 4) ) pipe_reg2 ( .clk (clk), .reset (reset), .enable (enable && stg2_mask), - .data_in ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, mant_is_zero_s1, src_d_s1, dst_d_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1, renorm_shamt_s1}), - .data_out ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, frm_s2, fclass_s2, unpacked_mant_s2, unpacked_exp_s2, renorm_shamt_s2}) + .data_in ({is_itof_s1, is_ftoi_s1, is_f2f_s1, is_signed_s1, is_int64_s1, input_sign_s1, mant_is_zero_s1, src_d_s1, dst_d_s1, src_h_s1, dst_h_s1, frm_s1, fclass_s1, unpacked_mant_s1, unpacked_exp_s1, renorm_shamt_s1}), + .data_out ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, src_h_s2, dst_h_s2, frm_s2, fclass_s2, unpacked_mant_s2, unpacked_exp_s2, renorm_shamt_s2}) ); // ====================================================================== @@ -252,14 +296,15 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // the S_MAN_WIDTH field. LZC counts the high padding too: subtract it back // for float inputs, using the source format's mantissa width. wire [S_EXP_WIDTH-1:0] fp_mant_offset_s2 = S_EXP_WIDTH'(S_MAN_WIDTH - 1) - - (src_d_s2 ? S_EXP_WIDTH'(F64_MAN) : S_EXP_WIDTH'(F32_MAN)); + - (src_h_s2 ? S_EXP_WIDTH'(F16_MAN) : src_d_s2 ? S_EXP_WIDTH'(F64_MAN) : S_EXP_WIDTH'(F32_MAN)); + `UNUSED_VAR (src_d_s2) wire [S_MAN_WIDTH-1:0] norm_mant_s2 = unpacked_mant_s2 << renorm_shamt_s2; wire signed [S_EXP_WIDTH-1:0] norm_exp_s2 = unpacked_exp_s2 - S_EXP_WIDTH'({1'b0, renorm_shamt_s2}) + (is_itof_s2 ? S_EXP_WIDTH'(0) : fp_mant_offset_s2); // Stage 2 -> Stage 3 - wire is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3; + wire is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3; wire [INST_FRM_BITS-1:0] frm_s3; fclass_t fclass_s3; wire [S_MAN_WIDTH-1:0] norm_mant_s3; @@ -267,14 +312,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s3; // pre-renorm exp, needed for f2i_shamt VX_pipe_register #( - .DATAW (9 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + 2*S_EXP_WIDTH), + .DATAW (10 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + 2*S_EXP_WIDTH), .DEPTH (LATENCY > 3) ) pipe_reg3 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-5]), - .data_in ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, frm_s2, fclass_s2, norm_mant_s2, norm_exp_s2, unpacked_exp_s2}), - .data_out ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, frm_s3, fclass_s3, norm_mant_s3, norm_exp_s3, unpacked_exp_s3}) + .data_in ({is_itof_s2, is_ftoi_s2, is_f2f_s2, is_signed_s2, is_int64_s2, input_sign_s2, mant_is_zero_s2, src_d_s2, dst_d_s2, dst_h_s2, frm_s2, fclass_s2, norm_mant_s2, norm_exp_s2, unpacked_exp_s2}), + .data_out ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3, frm_s3, fclass_s3, norm_mant_s3, norm_exp_s3, unpacked_exp_s3}) ); // ====================================================================== @@ -285,15 +330,18 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] f2i_shamt_s3 = S_EXP_WIDTH'(S_MAN_WIDTH - 1) - unpacked_exp_s3; // Destination float bias and true biased destination exponent. - wire signed [S_EXP_WIDTH-1:0] dst_bias_s3 = dst_d_s3 ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); + wire signed [S_EXP_WIDTH-1:0] dst_bias_s3 = dst_h_s3 ? S_EXP_WIDTH'(F16_BIAS) + : dst_d_s3 ? S_EXP_WIDTH'(F64_BIAS) : S_EXP_WIDTH'(F32_BIAS); wire signed [S_EXP_WIDTH-1:0] dst_exp_s3 = norm_exp_s3 + dst_bias_s3; + wire signed [S_EXP_WIDTH-1:0] dst_inf_exp_s3 = dst_h_s3 ? S_EXP_WIDTH'(31) : S_EXP_WIDTH'(255); - // F2F narrowing (FCVT.S.D) can under/overflow the F32 range; widening - // (FCVT.D.S) is always exact, so it never does. Only the F32-dst finite - // nonzero case matters here. + // F2F narrowing (FCVT.S.D, FCVT.H.S, FCVT.H.D) can under/overflow the + // destination range; widening is always exact, so it never does. Only the + // F32/F16-dst finite nonzero case matters here. wire f2f_narrow_s3 = is_f2f_s3 & ~dst_d_s3 & ~fclass_s3.is_nan & ~fclass_s3.is_inf & ~fclass_s3.is_zero; wire f2f_uf_s3 = f2f_narrow_s3 & (dst_exp_s3 <= 0); // -> subnormal/zero - wire f2f_of_s3 = f2f_narrow_s3 & (dst_exp_s3 >= S_EXP_WIDTH'(255)); // -> overflow to inf + wire f2f_of_s3 = f2f_narrow_s3 & (dst_exp_s3 >= dst_inf_exp_s3); // -> overflow + wire f2f_tiny0_s3 = f2f_narrow_s3 & (dst_exp_s3 == 0); wire signed [S_EXP_WIDTH-1:0] denorm_sh_s3 = S_EXP_WIDTH'(1) - dst_exp_s3; // right-shift to denormalize // Alignment right-shift: F2I integer align, or F2F destination denormalization. @@ -312,8 +360,8 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [SUPER_EXP-1:0] final_exp_s3 = SUPER_EXP'(dst_exp_s3); // Stage 3 -> Stage 4 - wire is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4; - wire f2f_uf_s4, f2f_of_s4; + wire is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4; + wire f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4; wire [INST_FRM_BITS-1:0] frm_s4; fclass_t fclass_s4; wire [S_MAN_WIDTH-1:0] norm_mant_s4; @@ -322,14 +370,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s4; // true magnitude exponent (F2I overflow) VX_pipe_register #( - .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + SUPER_EXP + S_EXP_WIDTH), + .DATAW (13 + INST_FRM_BITS + $bits(fclass_t) + S_MAN_WIDTH + S_EXP_WIDTH + SUPER_EXP + S_EXP_WIDTH), .DEPTH (LATENCY > 2) ) pipe_reg4 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-4]), - .data_in ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, f2f_uf_s3, f2f_of_s3, frm_s3, fclass_s3, norm_mant_s3, align_shamt_s3, final_exp_s3, unpacked_exp_s3}), - .data_out ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, f2f_uf_s4, f2f_of_s4, frm_s4, fclass_s4, norm_mant_s4, align_shamt_s4, final_exp_s4, unpacked_exp_s4}) + .data_in ({is_itof_s3, is_ftoi_s3, is_f2f_s3, is_signed_s3, is_int64_s3, input_sign_s3, mant_is_zero_s3, src_d_s3, dst_d_s3, dst_h_s3, f2f_uf_s3, f2f_of_s3, f2f_tiny0_s3, frm_s3, fclass_s3, norm_mant_s3, align_shamt_s3, final_exp_s3, unpacked_exp_s3}), + .data_out ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4, f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4, frm_s4, fclass_s4, norm_mant_s4, align_shamt_s4, final_exp_s4, unpacked_exp_s4}) ); // ====================================================================== @@ -344,7 +392,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire sticky_bit_s4 = |aligned_mant_full_s4[S_MAN_WIDTH-2:0]; // Destination float mantissa width (I2F / F2F) - wire [5:0] dst_man_w_s4 = dst_d_s4 ? 6'(F64_MAN) : 6'(F32_MAN); + wire [5:0] dst_man_w_s4 = dst_h_s4 ? 6'(F16_MAN) : dst_d_s4 ? 6'(F64_MAN) : 6'(F32_MAN); // Kept significand = {hidden, dst_man}. The round (guard) bit is the first bit // BELOW the kept LSB; sticky ORs everything beneath the guard. (Folding the @@ -355,6 +403,12 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire [S_MAN_WIDTH-1:0] fp_sticky_mask_s4= (S_MAN_WIDTH'(1) << (fp_trunc_sh_s4 - 7'd1)) - 1; wire sticky_red_s4 = |(aligned_mant_s4 & fp_sticky_mask_s4) | guard_bit_s4 | round_bit_s4 | sticky_bit_s4; + wire [S_MAN_WIDTH-1:0] fine_trunc_s4 = norm_mant_s4 >> fp_trunc_sh_s4; + wire [S_MAN_WIDTH-1:0] fine_ones_s4 = (S_MAN_WIDTH'(1) << (dst_man_w_s4 + 6'd1)) - 1; + wire [2:0] fine_round_s4 = {fine_trunc_s4 == fine_ones_s4, + norm_mant_s4[LZC_RESULT_WIDTH'(fp_trunc_sh_s4 - 7'd1)], + |(norm_mant_s4 & fp_sticky_mask_s4)}; + logic [1:0] round_sticky_bits_s4; logic [S_MAN_WIDTH-1:0] pre_round_abs_s4; @@ -371,8 +425,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; end // Stage 4 -> Stage 5 - wire is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5; - wire f2f_uf_s5, f2f_of_s5; + wire is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, dst_h_s5; + wire f2f_uf_s5, f2f_of_s5, f2f_tiny0_s5; + wire [2:0] fine_round_s5; wire [INST_FRM_BITS-1:0] frm_s5; fclass_t fclass_s5; wire [1:0] round_sticky_bits_s5; @@ -381,14 +436,14 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; wire signed [S_EXP_WIDTH-1:0] unpacked_exp_s5; VX_pipe_register #( - .DATAW (11 + INST_FRM_BITS + $bits(fclass_t) + 2 + S_MAN_WIDTH + SUPER_EXP + S_EXP_WIDTH), + .DATAW (16 + INST_FRM_BITS + $bits(fclass_t) + 2 + S_MAN_WIDTH + SUPER_EXP + S_EXP_WIDTH), .DEPTH (LATENCY > 1) ) pipe_reg5 ( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-3]), - .data_in ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, f2f_uf_s4, f2f_of_s4, frm_s4, fclass_s4, round_sticky_bits_s4, pre_round_abs_s4, final_exp_s4, unpacked_exp_s4}), - .data_out ({is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, f2f_uf_s5, f2f_of_s5, frm_s5, fclass_s5, round_sticky_bits_s5, pre_round_abs_s5, final_exp_s5, unpacked_exp_s5}) + .data_in ({is_itof_s4, is_ftoi_s4, is_f2f_s4, is_signed_s4, is_int64_s4, input_sign_s4, mant_is_zero_s4, src_d_s4, dst_d_s4, dst_h_s4, f2f_uf_s4, f2f_of_s4, f2f_tiny0_s4, fine_round_s4, frm_s4, fclass_s4, round_sticky_bits_s4, pre_round_abs_s4, final_exp_s4, unpacked_exp_s4}), + .data_out ({is_itof_s5, is_ftoi_s5, is_f2f_s5, is_signed_s5, is_int64_s5, input_sign_s5, mant_is_zero_s5, src_d_s5, dst_d_s5, dst_h_s5, f2f_uf_s5, f2f_of_s5, f2f_tiny0_s5, fine_round_s5, frm_s5, fclass_s5, round_sticky_bits_s5, pre_round_abs_s5, final_exp_s5, unpacked_exp_s5}) ); // ====================================================================== @@ -425,7 +480,7 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; // ---- Float pack (I2F / F2F), per destination format ---- // F32 result, NaN-boxed when XLEN>32. F64 result fills the full width. - wire f32_man_ovf = rounded_abs_s5[F32_MAN + 1]; + wire f32_man_ovf = f2f_uf_s5 ? rounded_abs_s5[F32_MAN] : rounded_abs_s5[F32_MAN + 1]; wire [F32_EXP-1:0] fp32_exp = safe_dst_exp[F32_EXP-1:0] + F32_EXP'(f32_man_ovf); wire [F32_MAN-1:0] fp32_man = f32_man_ovf ? '0 : safe_rounded_abs[F32_MAN-1:0]; wire [31:0] fp_32_res = {rounded_sign_s5, fp32_exp, fp32_man}; @@ -434,52 +489,104 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; : `VX_CFG_XLEN'(fp_32_res); wire [`VX_CFG_XLEN-1:0] fp_dst_res; - wire f2f_nv_s5, f2f_nx_s5; - if (HAS_D) begin : g_fp_pack_d - wire f64_man_ovf = rounded_abs_s5[F64_MAN + 1]; - wire [F64_EXP-1:0] fp64_exp = safe_dst_exp[F64_EXP-1:0] + F64_EXP'(f64_man_ovf); - wire [F64_MAN-1:0] fp64_man = f64_man_ovf ? '0 : safe_rounded_abs[F64_MAN-1:0]; - wire [63:0] fp_64_res = {rounded_sign_s5, fp64_exp, fp64_man}; + wire f2f_nv_s5, f2f_nx_s5, f2f_of_out_s5, f2f_uf_out_s5; + if (HAS_D || HAS_H) begin : g_fp_pack_f2f + wire [`VX_CFG_XLEN-1:0] fp64_boxed; + if (HAS_D) begin : g_fp64 + wire f64_man_ovf = rounded_abs_s5[F64_MAN + 1]; + wire [F64_EXP-1:0] fp64_exp = safe_dst_exp[F64_EXP-1:0] + F64_EXP'(f64_man_ovf); + wire [F64_MAN-1:0] fp64_man = f64_man_ovf ? '0 : safe_rounded_abs[F64_MAN-1:0]; + assign fp64_boxed = `VX_CFG_XLEN'({rounded_sign_s5, fp64_exp, fp64_man}); + end else begin : g_no_fp64 + assign fp64_boxed = '0; + end + + wire f16_man_ovf = f2f_uf_s5 ? rounded_abs_s5[F16_MAN] : rounded_abs_s5[F16_MAN + 1]; + wire [F16_EXP-1:0] fp16_exp = safe_dst_exp[F16_EXP-1:0] + F16_EXP'(f16_man_ovf); + wire [F16_MAN-1:0] fp16_man = f16_man_ovf ? '0 : safe_rounded_abs[F16_MAN-1:0]; + wire [15:0] fp_16_res = {rounded_sign_s5, fp16_exp, fp16_man}; + + wire [`VX_CFG_XLEN-1:0] qnan_d = `VX_CFG_XLEN'(64'h7FF8000000000000); + wire [`VX_CFG_XLEN-1:0] inf_d = `VX_CFG_XLEN'({input_sign_s5, 11'h7FF, 52'd0}); + wire [`VX_CFG_XLEN-1:0] zero_d = `VX_CFG_XLEN'({input_sign_s5, 63'd0}); + + wire [31:0] qnan_s32 = 32'h7FC00000; + wire [31:0] inf_s32 = {input_sign_s5, 8'hFF, 23'd0}; + wire [31:0] zero_s32 = {input_sign_s5, 31'd0}; + wire [31:0] max_s32 = {input_sign_s5, 8'hFE, {23{1'b1}}}; - // Canonical specials for F2F (NaN/Inf/Zero propagation in float domain) - wire [31:0] f2f_qnan_32 = 32'h7FC00000; - wire [31:0] f2f_inf_32 = {rounded_sign_s5, 8'hFF, 23'd0}; + wire [15:0] qnan_h16 = 16'h7E00; + wire [15:0] inf_h16 = {input_sign_s5, 5'h1F, 10'd0}; + wire [15:0] zero_h16 = {input_sign_s5, 15'd0}; + wire [15:0] max_h16 = {input_sign_s5, 5'h1E, {10{1'b1}}}; - wire [63:0] f2f_qnan_64 = 64'h7FF8000000000000; - wire [63:0] f2f_inf_64 = {rounded_sign_s5, 11'h7FF, 52'd0}; - wire [63:0] f2f_zero_64 = {input_sign_s5, 63'd0}; + wire [`VX_CFG_XLEN-1:0] box_s_qnan = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, qnan_s32} : `VX_CFG_XLEN'(qnan_s32); + wire [`VX_CFG_XLEN-1:0] box_s_inf = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, inf_s32} : `VX_CFG_XLEN'(inf_s32); + wire [`VX_CFG_XLEN-1:0] box_s_zero = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, zero_s32} : `VX_CFG_XLEN'(zero_s32); + wire [`VX_CFG_XLEN-1:0] box_s_max = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, max_s32} : `VX_CFG_XLEN'(max_s32); - // Normal (non-special) destination value by format. - wire [`VX_CFG_XLEN-1:0] normal_dst = dst_d_s5 ? `VX_CFG_XLEN'(fp_64_res) : f32_boxed; + wire [`VX_CFG_XLEN-1:0] box_h_res = {{(`VX_CFG_XLEN-16){1'b1}}, fp_16_res}; + wire [`VX_CFG_XLEN-1:0] box_h_qnan = {{(`VX_CFG_XLEN-16){1'b1}}, qnan_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_inf = {{(`VX_CFG_XLEN-16){1'b1}}, inf_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_zero = {{(`VX_CFG_XLEN-16){1'b1}}, zero_h16}; + wire [`VX_CFG_XLEN-1:0] box_h_max = {{(`VX_CFG_XLEN-16){1'b1}}, max_h16}; + + wire [`VX_CFG_XLEN-1:0] normal_dst = dst_d_s5 ? fp64_boxed : dst_h_s5 ? box_h_res : f32_boxed; - // F2F special-case results. wire is_nan_src = is_f2f_s5 & fclass_s5.is_nan; wire is_inf_src = is_f2f_s5 & fclass_s5.is_inf; wire is_zero_src = is_f2f_s5 & fclass_s5.is_zero; + wire is_fin_src = is_f2f_s5 & ~fclass_s5.is_nan & ~fclass_s5.is_inf & ~fclass_s5.is_zero; + + wire of_to_inf = (frm_s5 == INST_FRM_RNE) || (frm_s5 == INST_FRM_RMM) + || ((frm_s5 == INST_FRM_RUP) && ~input_sign_s5) + || ((frm_s5 == INST_FRM_RDN) && input_sign_s5); + wire of_post = is_fin_src & ~dst_d_s5 & ~f2f_of_s5 + & (dst_h_s5 ? (fp16_exp == 5'h1F) : (fp32_exp == 8'hFF)); + + wire [1:0] fine_rounded; + VX_fp_rounding #( + .DAT_WIDTH (2) + ) fine_rounding ( + .abs_value_i (2'b01), + .sign_i (input_sign_s5), + .round_sticky_bits_i (fine_round_s5[1:0]), + .rnd_mode_i (frm_s5), + .effective_subtraction_i (1'b0), + .abs_rounded_o (fine_rounded), + `UNUSED_PIN (sign_o), + `UNUSED_PIN (exact_zero_o) + ); + `UNUSED_VAR (fine_rounded[0]) + wire not_tiny = f2f_tiny0_s5 & fine_round_s5[2] & fine_rounded[1]; reg [`VX_CFG_XLEN-1:0] dst_r; always_comb begin - if (is_nan_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_qnan_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_qnan_32} : `VX_CFG_XLEN'(f2f_qnan_32)); - else if (is_inf_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_inf_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_inf_32} : `VX_CFG_XLEN'(f2f_inf_32)); - else if (is_zero_src) - dst_r = dst_d_s5 ? `VX_CFG_XLEN'(f2f_zero_64) - : ((`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, input_sign_s5, 31'd0} : `VX_CFG_XLEN'({input_sign_s5, 31'd0})); - else if (f2f_of_s5) // FCVT.S.D overflow -> F32 inf - dst_r = (`VX_CFG_XLEN > 32) ? {{(`VX_CFG_XLEN-32){1'b1}}, f2f_inf_32} : `VX_CFG_XLEN'(f2f_inf_32); - else + if (is_nan_src) begin + dst_r = dst_d_s5 ? qnan_d : dst_h_s5 ? box_h_qnan : box_s_qnan; + end else if (is_inf_src) begin + dst_r = dst_d_s5 ? inf_d : dst_h_s5 ? box_h_inf : box_s_inf; + end else if (is_zero_src) begin + dst_r = dst_d_s5 ? zero_d : dst_h_s5 ? box_h_zero : box_s_zero; + end else if (f2f_of_s5) begin + dst_r = dst_h_s5 ? (of_to_inf ? box_h_inf : box_h_max) + : (of_to_inf ? box_s_inf : box_s_max); + end else begin dst_r = normal_dst; + end end - assign fp_dst_res = dst_r; - assign f2f_nv_s5 = is_f2f_s5 & fclass_s5.is_signaling; - assign f2f_nx_s5 = is_f2f_s5 & ~fclass_s5.is_nan & ~fclass_s5.is_inf & ~fclass_s5.is_zero & (|round_sticky_bits_s5); + assign fp_dst_res = dst_r; + assign f2f_nv_s5 = is_f2f_s5 & fclass_s5.is_signaling; + assign f2f_of_out_s5 = f2f_of_s5 | of_post; + assign f2f_uf_out_s5 = f2f_uf_s5 & ~not_tiny; + assign f2f_nx_s5 = is_fin_src & ((|round_sticky_bits_s5) | f2f_of_out_s5); end else begin : g_fp_pack_s - assign fp_dst_res = f32_boxed; - assign f2f_nv_s5 = 1'b0; - assign f2f_nx_s5 = 1'b0; + `UNUSED_VAR ({dst_h_s5, f2f_of_s5, f2f_tiny0_s5, fine_round_s5}) + assign fp_dst_res = f32_boxed; + assign f2f_nv_s5 = 1'b0; + assign f2f_nx_s5 = 1'b0; + assign f2f_of_out_s5 = 1'b0; + assign f2f_uf_out_s5 = 1'b0; end // F2I saturation values for NaN/Inf inputs @@ -578,10 +685,9 @@ module VX_fcvt_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; res_val_64 = 64'(fp_dst_res); res_val_32 = fp_dst_res[31:0]; final_fflags_s5.NV = f2f_nv_s5; - final_fflags_s5.OF = f2f_of_s5; - final_fflags_s5.UF = f2f_uf_s5 & (|round_sticky_bits_s5); - final_fflags_s5.NX = is_f2f_s5 ? (f2f_nx_s5 | f2f_of_s5 | (f2f_uf_s5 & (|round_sticky_bits_s5))) - : (|round_sticky_bits_s5); + final_fflags_s5.OF = f2f_of_out_s5; + final_fflags_s5.UF = f2f_uf_out_s5 & (|round_sticky_bits_s5); + final_fflags_s5.NX = is_f2f_s5 ? f2f_nx_s5 : (|round_sticky_bits_s5); end end diff --git a/hw/rtl/fpu/VX_fncp_unit.sv b/hw/rtl/fpu/VX_fncp_unit.sv index 2526ddafbe..1cdd57db72 100644 --- a/hw/rtl/fpu/VX_fncp_unit.sv +++ b/hw/rtl/fpu/VX_fncp_unit.sv @@ -47,6 +47,11 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `UNUSED_VAR (fmt) wire is_d = HAS_D ? fmt[0] : 1'b0; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire is_h = fmt[1] && (op_type != INST_FPU_CMP) && (frm[2:1] == 2'b10); +`else + wire is_h = 1'b0; +`endif // F32 NaN-boxing (RISC-V spec): on a FLEN>32 register file a single-precision // operand is valid only if the upper bits are all ones; otherwise the operand @@ -149,6 +154,7 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( fclass_t a_fclass_s0, b_fclass_s0; wire a_smaller_s0, ab_equal_s0; wire is_d_s0; + wire is_h_s0; wire a32_boxed_s0; wire [3:0] op_mod = {(op_type == INST_FPU_CMP), frm}; @@ -158,14 +164,14 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( // a non-NaN-boxed single-precision source. Sign/class/ordering are already // derived above from the NaN-box-checked operands. VX_pipe_register #( - .DATAW (4 + 2 * FLEN + 1 + 1 + 2 * $bits(fclass_t) + 1 + 1 + 1 + 1), + .DATAW (4 + 2 * FLEN + 1 + 1 + 2 * $bits(fclass_t) + 1 + 1 + 1 + 1 + 1), .DEPTH (LATENCY > 0) ) pipe_reg0 ( .clk (clk), .reset (reset), .enable (enable && mask), - .data_in ({op_mod, dataa, datab, a_sign_sel, b_sign_sel, a_class_sel, b_class_sel, a_smaller_sel, ab_equal_sel, is_d, a32_boxed}), - .data_out ({op_mod_s0, dataa_s0, datab_s0, a_sign_s0, b_sign_s0, a_fclass_s0, b_fclass_s0, a_smaller_s0, ab_equal_s0, is_d_s0, a32_boxed_s0}) + .data_in ({op_mod, dataa, datab, a_sign_sel, b_sign_sel, a_class_sel, b_class_sel, a_smaller_sel, ab_equal_sel, is_d, is_h, a32_boxed}), + .data_out ({op_mod_s0, dataa_s0, datab_s0, a_sign_s0, b_sign_s0, a_fclass_s0, b_fclass_s0, a_smaller_s0, ab_equal_s0, is_d_s0, is_h_s0, a32_boxed_s0}) ); `UNUSED_VAR (b_fclass_s0) @@ -324,6 +330,16 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( assign result_xlen_s0 = result_s0; end + wire [`VX_CFG_XLEN-1:0] result_out_s0; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + wire [`VX_CFG_XLEN-1:0] result_h_s0 = op_mod_s0[0] ? {{(`VX_CFG_XLEN-16){1'b1}}, dataa_s0[15:0]} + : {{(`VX_CFG_XLEN-16){dataa_s0[15]}}, dataa_s0[15:0]}; + assign result_out_s0 = is_h_s0 ? result_h_s0 : result_xlen_s0; +`else + `UNUSED_VAR (is_h_s0) + assign result_out_s0 = result_xlen_s0; +`endif + wire fflags_NV; VX_pipe_register #( @@ -333,7 +349,7 @@ module VX_fncp_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .clk (clk), .reset (reset), .enable (enable && mask_pipe[LATENCY-2]), - .data_in ({result_xlen_s0, fflags_NV_s0}), + .data_in ({result_out_s0, fflags_NV_s0}), .data_out ({result, fflags_NV}) ); // NV, DZ, OF, UF, NX diff --git a/hw/rtl/fpu/VX_fpu_dpi.sv b/hw/rtl/fpu/VX_fpu_dpi.sv index 605e1a48a3..411751225a 100644 --- a/hw/rtl/fpu/VX_fpu_dpi.sv +++ b/hw/rtl/fpu/VX_fpu_dpi.sv @@ -77,7 +77,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( reg [FPC_BITS-1:0] core_select; reg is_fadd, is_fsub, is_fmul, is_fmadd, is_fmsub, is_fnmadd, is_fnmsub; - reg is_div, is_fcmp, is_itof, is_utof, is_ftoi, is_ftou, is_f2f; + reg is_div, is_fcmp, is_itof, is_utof, is_ftoi, is_ftou, is_f2f, is_f2h; reg [NUM_LANES-1:0][63:0] operands [3]; @@ -107,6 +107,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( is_ftoi = 0; is_ftou = 0; is_f2f = 0; + is_f2h = 0; case (op_type) INST_FPU_ADD: begin core_select = FPU_FMA; is_fadd = ~i_fmt; is_fsub = i_fmt; end @@ -121,6 +122,7 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( INST_FPU_I2F: begin core_select = FPU_CVT; is_itof = 1; end INST_FPU_U2F: begin core_select = FPU_CVT; is_utof = 1; end INST_FPU_F2F: begin core_select = FPU_CVT; is_f2f = 1; end + INST_FPU_F2H: begin core_select = FPU_CVT; is_f2h = 1; end default: begin core_select = FPU_NCP; end endcase end @@ -293,26 +295,29 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire fcvt_ready = per_core_ready_out[FPU_CVT] || ~per_core_valid_out[FPU_CVT]; wire fcvt_fire = fcvt_valid && fcvt_ready; + wire [1:0] f2f_dst_fmt = is_f2h ? 2'd2 : {1'b0, f_fmt}; + wire [1:0] f2f_src_fmt = is_f2h ? {1'b0, f_fmt} : (i_fmt ? 2'd2 : {1'b0, ~f_fmt}); + always @(*) begin for (integer i = 0; i < NUM_LANES; ++i) begin dpi_itof (fcvt_fire, int'(f_fmt), int'(i_fmt), operands[0][i], frm, result_itof[i], fflags_itof[i]); dpi_utof (fcvt_fire, int'(f_fmt), int'(i_fmt), operands[0][i], frm, result_utof[i], fflags_utof[i]); dpi_ftoi (fcvt_fire, int'(i_fmt), int'(f_fmt), operands[0][i], frm, result_ftoi[i], fflags_ftoi[i]); dpi_ftou (fcvt_fire, int'(i_fmt), int'(f_fmt), operands[0][i], frm, result_ftou[i], fflags_ftou[i]); - dpi_f2f (fcvt_fire, int'(f_fmt), int'($bits(f_fmt)'(~f_fmt)), operands[0][i], frm, result_f2f[i], fflags_f2f[i]); + dpi_f2f (fcvt_fire, int'(f2f_dst_fmt), int'(f2f_src_fmt), operands[0][i], frm, result_f2f[i], fflags_f2f[i]); result_fcvt[i] = is_itof ? result_itof[i][`VX_CFG_XLEN-1:0] : is_utof ? result_utof[i][`VX_CFG_XLEN-1:0] : is_ftoi ? result_ftoi[i][`VX_CFG_XLEN-1:0] : is_ftou ? result_ftou[i][`VX_CFG_XLEN-1:0] : - is_f2f ? result_f2f[i][`VX_CFG_XLEN-1:0] : + (is_f2f || is_f2h) ? result_f2f[i][`VX_CFG_XLEN-1:0] : '0; fflags_fcvt[i] = is_itof ? fflags_itof[i] : is_utof ? fflags_utof[i] : is_ftoi ? fflags_ftoi[i] : is_ftou ? fflags_ftou[i] : - is_f2f ? fflags_f2f[i] : + (is_f2f || is_f2h) ? fflags_f2f[i] : '0; end end @@ -376,6 +381,12 @@ module VX_fpu_dpi import VX_gpu_pkg::*, VX_fpu_pkg::*; #( dpi_fsgnjx (fncp_fire, int'(f_fmt), operands[0][i], operands[1][i], result_fsgnjx[i]); result_fmvx[i] = f_fmt ? operands[0][i] : 64'($signed(operands[0][i][31:0])); // sign-extension result_fmvf[i] = f_fmt ? operands[0][i] : (operands[0][i] | 64'hffffffff00000000); // nan-boxing + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (i_fmt) begin + result_fmvx[i] = 64'($signed(operands[0][i][15:0])); + result_fmvf[i] = operands[0][i] | 64'hffffffffffff0000; + end + `endif end end diff --git a/hw/rtl/fpu/VX_fpu_dsp.sv b/hw/rtl/fpu/VX_fpu_dsp.sv index 240e52cd59..d1a080cc4d 100644 --- a/hw/rtl/fpu/VX_fpu_dsp.sv +++ b/hw/rtl/fpu/VX_fpu_dsp.sv @@ -96,7 +96,7 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_FPCORES-1:0] per_core_ready_out; // Decode fpu core type - wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F) ? FPCORES_BITS'(FPU_CVT) + wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F || op_type == INST_FPU_F2H) ? FPCORES_BITS'(FPU_CVT) : FPCORES_BITS'(op_type[3:2]); VX_stream_switch #( @@ -590,17 +590,21 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [INST_FMT_BITS-1:0] pe_fmt = pe_shared[INST_FRM_BITS+:INST_FMT_BITS]; wire [INST_FPU_BITS-1:0] pe_op = pe_shared[INST_FRM_BITS+INST_FMT_BITS+:INST_FPU_BITS]; - wire is_f2f = (pe_op == INST_FPU_F2F); + wire is_f2h = (pe_op == INST_FPU_F2H); + wire is_f2f = (pe_op == INST_FPU_F2F) || is_f2h; wire is_itof = pe_op[1] & ~is_f2f; wire is_ftoi = ~pe_op[1] & ~is_f2f; wire is_signed = ~pe_op[0]; wire is_int64 = pe_fmt[1]; - wire src_fmt = is_f2f ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format + wire src_fmt = (is_f2f && !is_f2h) ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format wire dst_fmt = pe_fmt[0]; VX_fcvt_unit #( .LATENCY (CVT_LATENCY), .FLEN (`VX_CFG_FLEN), + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + .HALF_ENABLE (1), + `endif .OUT_REG (1) ) fcvt_unit ( .clk (clk), @@ -615,6 +619,8 @@ module VX_fpu_dsp import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .is_int64 (is_int64), .src_fmt (src_fmt), .dst_fmt (dst_fmt), + .src_h (is_f2f && !is_f2h && pe_fmt[1]), + .dst_h (is_f2h), .dataa (pe_data_in[i][0+:`VX_CFG_XLEN]), .result (pe_data_out[i][0+:`VX_CFG_XLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/hw/rtl/fpu/VX_fpu_fpnew.sv b/hw/rtl/fpu/VX_fpu_fpnew.sv index 22ddf4f347..8ce3b17b72 100644 --- a/hw/rtl/fpu/VX_fpu_fpnew.sv +++ b/hw/rtl/fpu/VX_fpu_fpnew.sv @@ -56,20 +56,26 @@ module VX_fpu_fpnew localparam LATENCY_FDIVSQRT = `MAX(`VX_CFG_FDIV_LATENCY, `VX_CFG_FSQRT_LATENCY); localparam RSP_DATAW = (NUM_LANES * `VX_CFG_XLEN) + 1 + $bits(fflags_t) + TAG_WIDTH; +`ifdef VX_CFG_EXT_ZFHMIN_ENABLE + localparam [4:0] FP16_FMT_MASK = 5'b00100; +`else + localparam [4:0] FP16_FMT_MASK = 5'b00000; +`endif + localparam fpnew_pkg::fpu_features_t FPU_FEATURES = '{ Width: unsigned'(`VX_CFG_XLEN), EnableVectors: 1'b0, `ifdef VX_CFG_XLEN_64 EnableNanBox: 1'b1, `ifdef VX_CFG_FLEN_64 - FpFmtMask: 5'b11000, + FpFmtMask: 5'b11000 | FP16_FMT_MASK, `else - FpFmtMask: 5'b11000, // FP32+FP64 enabled to avoid CVT bug in FpNew + FpFmtMask: 5'b11000 | FP16_FMT_MASK, // FP32+FP64 enabled to avoid CVT bug in FpNew `endif IntFmtMask: 4'b0011 `else - EnableNanBox: 1'b0, - FpFmtMask: 5'b10000, + EnableNanBox: (FP16_FMT_MASK != 0), + FpFmtMask: 5'b10000 | FP16_FMT_MASK, IntFmtMask: 4'b0010 `endif }; @@ -142,7 +148,25 @@ module VX_fpu_fpnew INST_FPU_NMADD: begin fpu_op = fpnew_pkg::FNMSUB; fpu_op_mod = ~fmt[1]; end INST_FPU_DIV: begin fpu_op = fpnew_pkg::DIV; end INST_FPU_SQRT: begin fpu_op = fpnew_pkg::SQRT; end - `ifdef VX_CFG_FLEN_64 + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + INST_FPU_F2F: begin + fpu_op = fpnew_pkg::F2F; + `ifdef VX_CFG_FLEN_64 + fpu_src_fmt = fmt[1] ? fpnew_pkg::FP16 : (fmt[0] ? fpnew_pkg::FP32 : fpnew_pkg::FP64); + `else + fpu_src_fmt = fpnew_pkg::FP16; + `endif + end + INST_FPU_F2H: begin + fpu_op = fpnew_pkg::F2F; + fpu_dst_fmt = fpnew_pkg::FP16; + `ifdef VX_CFG_FLEN_64 + fpu_src_fmt = fmt[0] ? fpnew_pkg::FP64 : fpnew_pkg::FP32; + `else + fpu_src_fmt = fpnew_pkg::FP32; + `endif + end + `elsif VX_CFG_FLEN_64 INST_FPU_F2F: begin fpu_op = fpnew_pkg::F2F; fpu_src_fmt = fmt[0] ? fpnew_pkg::FP32 : fpnew_pkg::FP64; end `endif INST_FPU_F2I, @@ -154,7 +178,15 @@ module VX_fpu_fpnew case (frm) 0,1,2: begin fpu_op = fpnew_pkg::SGNJ; fpu_rnd = {1'b0, frm[1:0]}; fpu_has_fflags = 0; end // FSGNJ 3: begin fpu_op = fpnew_pkg::CLASSIFY; fpu_has_fflags = 0; end // CLASS - 4,5: begin fpu_op = fpnew_pkg::SGNJ; fpu_rnd = 3'b011; fpu_op_mod = ~frm[0]; fpu_has_fflags = 0; end // FMV.X.W, FMV.W.X + 4,5: begin + fpu_op = fpnew_pkg::SGNJ; fpu_rnd = 3'b011; fpu_op_mod = ~frm[0]; fpu_has_fflags = 0; + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + if (fmt[1]) begin + fpu_src_fmt = fpnew_pkg::FP16; + fpu_dst_fmt = fpnew_pkg::FP16; + end + `endif + end 6,7: begin fpu_op = fpnew_pkg::MINMAX; fpu_rnd = {2'b00, frm[0]}; end // MIN, MAX endcase end diff --git a/hw/rtl/fpu/VX_fpu_std.sv b/hw/rtl/fpu/VX_fpu_std.sv index 5fed9717f4..81a24a8e52 100644 --- a/hw/rtl/fpu/VX_fpu_std.sv +++ b/hw/rtl/fpu/VX_fpu_std.sv @@ -87,7 +87,7 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [NUM_FPCORES-1:0] per_core_ready_out; // Decode FPU core selector - wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F) ? FPCORES_BITS'(FPU_CVT) + wire [FPCORES_BITS-1:0] core_select = (op_type == INST_FPU_F2F || op_type == INST_FPU_F2H) ? FPCORES_BITS'(FPU_CVT) : FPCORES_BITS'(op_type[3:2]); VX_stream_switch #( @@ -413,17 +413,21 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( wire [INST_FMT_BITS-1:0] pe_fmt = pe_shared[INST_FRM_BITS+:INST_FMT_BITS]; wire [INST_FPU_BITS-1:0] pe_op = pe_shared[INST_FRM_BITS+INST_FMT_BITS+:INST_FPU_BITS]; - wire is_f2f = (pe_op == INST_FPU_F2F); + wire is_f2h = (pe_op == INST_FPU_F2H); + wire is_f2f = (pe_op == INST_FPU_F2F) || is_f2h; wire is_itof = pe_op[1] & ~is_f2f; wire is_ftoi = ~pe_op[1] & ~is_f2f; wire is_signed = ~pe_op[0]; wire is_int64 = pe_fmt[1]; - wire src_fmt = is_f2f ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format + wire src_fmt = (is_f2f && !is_f2h) ? ~pe_fmt[0] : pe_fmt[0]; // F2F: src is the other format wire dst_fmt = pe_fmt[0]; VX_fcvt_unit #( .LATENCY (CVT_LATENCY), .FLEN (`VX_CFG_FLEN), + `ifdef VX_CFG_EXT_ZFHMIN_ENABLE + .HALF_ENABLE (1), + `endif .OUT_REG (1) ) fcvt_unit ( .clk (clk), @@ -438,6 +442,8 @@ module VX_fpu_std import VX_gpu_pkg::*, VX_fpu_pkg::*; #( .is_int64 (is_int64), .src_fmt (src_fmt), .dst_fmt (dst_fmt), + .src_h (is_f2f && !is_f2h && pe_fmt[1]), + .dst_h (is_f2h), .dataa (pe_data_in[i][0+:`VX_CFG_XLEN]), .result (pe_data_out[i][0+:`VX_CFG_XLEN]), .fflags (pe_data_out[i][`VX_CFG_XLEN+:`FP_FLAGS_BITS]) diff --git a/hw/rtl/libs/VX_rom.sv b/hw/rtl/libs/VX_rom.sv new file mode 100644 index 0000000000..424d81a7a1 --- /dev/null +++ b/hw/rtl/libs/VX_rom.sv @@ -0,0 +1,58 @@ +`include "VX_platform.vh" + +`TRACING_OFF +module VX_rom #( + parameter DATAW = 1, + parameter SIZE = 1, + parameter OUT_REG = 0, + parameter [SIZE*DATAW-1:0] INIT_DATA = '0, + parameter ADDRW = `LOG2UP(SIZE) +) ( + input wire clk, + input wire read, + input wire [ADDRW-1:0] addr, + output wire [DATAW-1:0] rdata +); +`ifdef SYNTHESIS + localparam BLOCK = (OUT_REG != 0) && `FORCE_BRAM(SIZE, DATAW); +`else + localparam BLOCK = 0; +`endif + + if (BLOCK) begin : g_block + `USE_BLOCK_ROM reg [DATAW-1:0] mem [0:SIZE-1]; + reg [DATAW-1:0] rdata_r; + initial begin + for (integer i = 0; i < SIZE; ++i) begin + mem[i] = INIT_DATA[i * DATAW +: DATAW]; + end + end + always @(posedge clk) begin + if (read) begin + rdata_r <= mem[addr]; + end + end + assign rdata = rdata_r; + end else begin : g_logic + reg [DATAW-1:0] mem [0:SIZE-1]; + initial begin + for (integer i = 0; i < SIZE; ++i) begin + mem[i] = INIT_DATA[i * DATAW +: DATAW]; + end + end + if (OUT_REG != 0) begin : g_out_reg + reg [DATAW-1:0] rdata_r; + always @(posedge clk) begin + if (read) begin + rdata_r <= mem[addr]; + end + end + assign rdata = rdata_r; + end else begin : g_no_out_reg + `UNUSED_VAR ({clk, read}) + assign rdata = mem[addr]; + end + end + +endmodule +`TRACING_ON diff --git a/hw/rtl/mufu/VX_mufu_pe.sv b/hw/rtl/mufu/VX_mufu_pe.sv new file mode 100644 index 0000000000..3a57f48ac3 --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_pe.sv @@ -0,0 +1,371 @@ +`include "VX_define.vh" + +module VX_mufu_pe import VX_gpu_pkg::*; ( + input wire clk, + input wire reset, + input wire enable, + input wire [2:0] fn, + input wire [31:0] dataa, + output wire [31:0] result +); + `STATIC_ASSERT (`VX_MUFU_LATENCY == 7, ("VX_mufu_pe: the datapath is 7 stages, VX_MUFU_LATENCY=%0d", `VX_MUFU_LATENCY)) + + localparam TBL_RCP = 3'd0; + localparam TBL_RSQRT = 3'd1; + localparam TBL_EX2 = 3'd2; + localparam TBL_LG2 = 3'd3; + localparam TBL_SIN = 3'd4; + localparam TBL_TANH = 3'd5; + + localparam F32_QNAN = 32'h7fc00000; + localparam F32_POS_INF = 32'h7f800000; + localparam F32_NEG_INF = 32'hff800000; + localparam F32_ONE = 32'h3f800000; + + wire x_s = dataa[31]; + wire [7:0] x_e = dataa[30:23]; + wire [22:0] x_m = dataa[22:0]; + wire [23:0] x_sig = {1'b1, x_m}; + + wire x_zero = (x_e == 8'd0); + wire x_inf = (x_e == 8'hff) && (x_m == 23'd0); + wire x_nan = (x_e == 8'hff) && (x_m != 23'd0); + + wire signed [9:0] x_k = 10'($signed({2'b00, x_e})) - 10'sd127; + + wire [3:0] ex2_lsh = 4'(x_e - 8'd120); + wire [7:0] ex2_rsh = 8'd120 - x_e; + wire [37:0] ex2_mag = (x_e >= 8'd120) ? (38'(x_sig) << ex2_lsh) + : ((ex2_rsh < 8'd24) ? 38'(x_sig >> ex2_rsh[4:0]) : 38'd0); + wire [38:0] ex2_x = x_s ? -{1'b0, ex2_mag} : {1'b0, ex2_mag}; + + wire [7:0] trig_lsh = x_e - 8'd119; + wire [7:0] trig_rsh = 8'd119 - x_e; + wire is_sin = (fn == 3'(INST_MUFU_SINPI)); + wire [31:0] trig_u0 = (x_e >= 8'd119) ? ((trig_lsh < 8'd32) ? (32'(x_sig) << trig_lsh[4:0]) : 32'd0) + : ((trig_rsh < 8'd24) ? 32'(x_sig >> trig_rsh[4:0]) : 32'd0); + wire [31:0] trig_u = is_sin ? trig_u0 : (trig_u0 + 32'h40000000); + wire trig_odd = trig_u[30]; + wire [29:0] trig_t = trig_odd ? ~trig_u[29:0] : trig_u[29:0]; + wire trig_neg = trig_u[31] ^ (is_sin && x_s); + wire trig_one = trig_odd && (trig_u[29:0] == 30'd0); + `UNUSED_VAR ({ex2_x[5:0], trig_t[5:0]}) + + reg r_special; + reg [31:0] r_sval; + reg [2:0] r_tbl; + reg [6:0] r_idx; + reg [17:0] r_xl; + reg signed [9:0] r_exp; + reg r_neg; + reg r_zsign; + reg r_lg2; + + always @(*) begin + r_special = 1'b0; + r_sval = 32'd0; + r_tbl = TBL_RCP; + r_idx = x_m[22:16]; + r_xl = {x_m[15:0], 2'b00}; + r_exp = 10'sd0; + r_neg = 1'b0; + r_zsign = 1'b0; + r_lg2 = 1'b0; + case (fn) + 3'(INST_MUFU_RCP): begin + r_neg = x_s; + r_exp = 10'sd127 - 10'($signed({2'b00, x_e})); + if (x_inf) begin + r_special = 1'b1; + r_sval = {x_s, 31'd0}; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = x_s ? F32_NEG_INF : F32_POS_INF; + end + end + 3'(INST_MUFU_RSQRT): begin + r_tbl = TBL_RSQRT; + r_idx = {x_k[0], x_m[22:17]}; + r_xl = {x_m[16:0], 1'b0}; + r_exp = -(x_k >>> 1); + if (x_zero) begin + r_special = 1'b1; + r_sval = x_s ? F32_NEG_INF : F32_POS_INF; + end else if (x_s) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_inf) begin + r_special = 1'b1; + r_sval = 32'd0; + end + end + 3'(INST_MUFU_EX2): begin + r_tbl = TBL_EX2; + r_idx = {1'b0, ex2_x[29:24]}; + r_xl = ex2_x[23:6]; + r_exp = 10'($signed(ex2_x[38:30])); + if (x_inf) begin + r_special = 1'b1; + r_sval = x_s ? 32'd0 : F32_POS_INF; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = F32_ONE; + end else if (x_e >= 8'd134) begin + r_special = 1'b1; + r_sval = x_s ? 32'd0 : F32_POS_INF; + end + end + 3'(INST_MUFU_LG2): begin + r_tbl = TBL_LG2; + r_idx = {1'b0, x_m[22:17]}; + r_xl = {x_m[16:0], 1'b0}; + r_lg2 = 1'b1; + if (x_zero) begin + r_special = 1'b1; + r_sval = F32_NEG_INF; + end else if (x_s) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_inf) begin + r_special = 1'b1; + r_sval = F32_POS_INF; + end + end + 3'(INST_MUFU_SINPI), + 3'(INST_MUFU_COSPI): begin + r_tbl = TBL_SIN; + r_idx = {1'b0, trig_t[29:24]}; + r_xl = trig_t[23:6]; + r_neg = trig_neg; + r_zsign = is_sin && x_s; + if (x_inf) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = is_sin ? {x_s, 31'd0} : F32_ONE; + end else if (trig_one) begin + r_special = 1'b1; + r_sval = {trig_neg, F32_ONE[30:0]}; + end + end + 3'(INST_MUFU_TANH): begin + r_tbl = TBL_TANH; + r_idx = {1'b0, 3'(x_e - 8'd122), x_m[22:20]}; + r_xl = x_m[19:2]; + r_exp = x_k + 10'sd4; + r_neg = x_s; + if (x_e >= 8'd130) begin + r_special = 1'b1; + r_sval = {x_s, F32_ONE[30:0]}; + end else if (x_zero) begin + r_special = 1'b1; + r_sval = {x_s, 31'd0}; + end else if (x_e < 8'd122) begin + r_special = 1'b1; + r_sval = dataa; + end + end + default: begin + r_special = 1'b1; + r_sval = 32'd0; + end + endcase + if (x_nan) begin + r_special = 1'b1; + r_sval = F32_QNAN; + end + end + + localparam MISC_W = 1 + 32 + 10 + 1 + 1 + 1 + 10; + + wire [MISC_W-1:0] s1_misc, s2_misc, s3_misc, s4_misc; + wire [2:0] s1_tbl; + wire [6:0] s1_idx; + wire [17:0] s1_xl, s2_xl, s3_xl; + + VX_pipe_register #( + .DATAW (MISC_W + 3 + 7 + 18) + ) s1_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({r_special, r_sval, r_exp, r_neg, r_zsign, r_lg2, x_k, r_tbl, r_idx, r_xl}), + .data_out ({s1_misc, s1_tbl, s1_idx, s1_xl}) + ); + + wire [31:0] rom_a0; + wire signed [26:0] rom_a1; + wire signed [19:0] rom_a2; + + VX_mufu_rom rom ( + .clk (clk), + .enable (enable), + .tbl (s1_tbl), + .idx (s1_idx), + .a0 (rom_a0), + .a1 (rom_a1), + .a2 (rom_a2) + ); + + VX_pipe_register #( + .DATAW (MISC_W + 18) + ) s2_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s1_misc, s1_xl}), + .data_out ({s2_misc, s2_xl}) + ); + + wire [35:0] xl_sq = 36'(s2_xl) * 36'(s2_xl); + `UNUSED_VAR (xl_sq[17:0]) + + wire [17:0] s3_sq; + + VX_pipe_register #( + .DATAW (MISC_W + 18 + 18) + ) s3_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s2_misc, s2_xl, xl_sq[35:18]}), + .data_out ({s3_misc, s3_xl, s3_sq}) + ); + + wire signed [45:0] s4_p1; + wire signed [38:0] s4_p2; + + VX_multiplier #( + .A_WIDTH (27), + .B_WIDTH (19), + .R_WIDTH (46), + .SIGNED (1), + .LATENCY (1) + ) mul1 ( + .clk (clk), + .enable (enable), + .dataa (rom_a1), + .datab ({1'b0, s3_xl}), + .result (s4_p1) + ); + + VX_multiplier #( + .A_WIDTH (20), + .B_WIDTH (19), + .R_WIDTH (39), + .SIGNED (1), + .LATENCY (1) + ) mul2 ( + .clk (clk), + .enable (enable), + .dataa (rom_a2), + .datab ({1'b0, s3_sq}), + .result (s4_p2) + ); + + wire [31:0] s4_a0; + + VX_pipe_register #( + .DATAW (MISC_W + 32) + ) s4_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s3_misc, rom_a0}), + .data_out ({s4_misc, s4_a0}) + ); + + wire s4_special, s4_neg, s4_zsign, s4_lg2; + wire [31:0] s4_sval; + wire signed [9:0] s4_exp, s4_k; + assign {s4_special, s4_sval, s4_exp, s4_neg, s4_zsign, s4_lg2, s4_k} = s4_misc; + + wire signed [27:0] t1 = 28'(s4_p1 >>> 18); + wire signed [20:0] t2 = 21'(s4_p2 >>> 18); + wire signed [34:0] p_sum = $signed({3'b000, s4_a0}) + 35'(t1) + 35'(t2); + wire [33:0] p_pos = p_sum[34] ? 34'd0 : p_sum[33:0]; + wire signed [40:0] lg2_sum = 41'($signed({s4_k, 30'd0})) + $signed({7'd0, p_pos}); + wire lg2_neg = lg2_sum[40]; + wire [39:0] lg2_mag = 40'(lg2_neg ? -lg2_sum : lg2_sum); + + wire [39:0] mag = s4_lg2 ? lg2_mag : 40'(p_pos); + wire neg = s4_lg2 ? lg2_neg : s4_neg; + wire signed [9:0] exp_adj = s4_lg2 ? 10'sd0 : s4_exp; + wire zsign = s4_lg2 ? 1'b0 : s4_zsign; + + wire s5_special, s5_neg, s5_zsign; + wire [31:0] s5_sval; + wire signed [9:0] s5_exp; + wire [39:0] s5_mag; + + VX_pipe_register #( + .DATAW (1 + 32 + 10 + 1 + 1 + 40) + ) s5_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s4_special, s4_sval, exp_adj, neg, zsign, mag}), + .data_out ({s5_special, s5_sval, s5_exp, s5_neg, s5_zsign, s5_mag}) + ); + + wire [5:0] lz; + wire mag_nz; + + VX_lzc #( + .N (40) + ) lzc ( + .data_in (s5_mag), + .data_out (lz), + .valid_out (mag_nz) + ); + + wire [39:0] norm = s5_mag << lz; + wire signed [11:0] bexp_n = 12'sd136 - 12'($signed({1'b0, lz})) + 12'(s5_exp); + `UNUSED_VAR (norm[14:0]) + + wire s6_special, s6_neg, s6_zsign, s6_nz; + wire [31:0] s6_sval; + wire signed [11:0] s6_bexp; + wire [24:0] s6_sig; + + VX_pipe_register #( + .DATAW (1 + 32 + 1 + 1 + 1 + 12 + 25) + ) s6_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s5_special, s5_sval, s5_neg, s5_zsign, mag_nz, bexp_n, norm[39:15]}), + .data_out ({s6_special, s6_sval, s6_neg, s6_zsign, s6_nz, s6_bexp, s6_sig}) + ); + + wire [24:0] sig_r = {1'b0, s6_sig[24:1]} + 25'(s6_sig[0]); + wire signed [11:0] bexp = s6_bexp + 12'(sig_r[24]); + wire [22:0] mant = sig_r[24] ? sig_r[23:1] : sig_r[22:0]; + + reg [31:0] packed_res; + always @(*) begin + if (s6_special) begin + packed_res = s6_sval; + end else if (~s6_nz) begin + packed_res = {s6_zsign, 31'd0}; + end else if (bexp >= 12'sd255) begin + packed_res = {s6_neg, F32_POS_INF[30:0]}; + end else if (bexp <= 12'sd0) begin + packed_res = {s6_neg, 31'd0}; + end else begin + packed_res = {s6_neg, bexp[7:0], mant}; + end + end + + VX_pipe_register #( + .DATAW (32) + ) s7_reg ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (packed_res), + .data_out (result) + ); + +endmodule diff --git a/hw/rtl/mufu/VX_mufu_rom.sv b/hw/rtl/mufu/VX_mufu_rom.sv new file mode 100644 index 0000000000..6a3120288a --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_rom.sv @@ -0,0 +1,575 @@ +`include "VX_platform.vh" + +module VX_mufu_rom ( + input wire clk, + input wire enable, + input wire [2:0] tbl, + input wire [6:0] idx, + output wire [31:0] a0, + output wire signed [26:0] a1, + output wire signed [19:0] a2 +); + localparam DATAW = 72; + localparam SIZE = 512; + + localparam [SIZE*DATAW-1:0] ROM_DATA = { + 72'h01fffff0000017fffe, + 72'h01ffffc800003ffffc, + 72'h01ffff68000097fff4, + 72'h01fffe68000187ffe0, + 72'h01fffba800041fffa9, + 72'h01fff420000b2fff13, + 72'h01ffdfc0001e57fd7c, + 72'h01ffa86000525ff92a, + 72'h03fede58008e17f8e9, + 72'h03fe229800ea17f453, + 72'h03fced6001817fecc8, + 72'h03faf050027a5fe06a, + 72'h03f7ab280412afcc31, + 72'h03f24cc006ae6fab4b, + 72'h03e982780aecbf7643, + 72'h03db2e7011c4d72203, + 72'h07a1e40016e18f6344, + 72'h0787f1c01d014f3b68, + 72'h07671e2824a3770b06, + 72'h073dc1082e1336d146, + 72'h0709e5a8399e3e8de0, + 72'h06c94c50478cb64190, + 72'h0679767058143deeb1, + 72'h0617c2406b453d99fc, + 72'h0bbec2a075fefeaec1, + 72'h0b433350811b669c9c, + 72'h0abc48a08cb58e8cf6, + 72'h0a29927898b26e80b0, + 72'h098abf50a4ef9e78ba, + 72'h08dfa408b1430e7603, + 72'h08284348bd7b0e7982, + 72'h0764d478c95fce83fb, + 72'h0dfd7340cf15af4856, + 72'h0d2b94d0d4a7774ddb, + 72'h0c543ee0da04f75465, + 72'h0b77a9b8df25ff5bf3, + 72'h0a9615e8e402776485, + 72'h09afcc00e8925f6e17, + 72'h08c51c78eccdf778a0, + 72'h07d65f48f0add78416, + 72'h0ebb32e0f2723fc69f, + 72'h0dc7e820f423dfc9ce, + 72'h0cd2f8d8f5bb37cd17, + 72'h0bdc7fc0f7378fd077, + 72'h0ae49838f89827d3ee, + 72'h09eb5e38f9dc67d77a, + 72'h08f0ee78fb03afdb19, + 72'h07f56640fc0d67deca, + 72'h0eee84b0fc84e7f0cd, + 72'h0df1c6d8fcf6f7f1bf, + 72'h0cf49ac8fd615ff2b3, + 72'h0bf70828fdc41ff3a8, + 72'h0af916a8fe1f1ff4a0, + 72'h09face00fe725ff598, + 72'h08fc3600febdd7f692, + 72'h07fd5680ff016ff78d, + 72'h0efb9cb0ff1fb7fc25, + 72'h0dfc6e90ff3c97fc64, + 72'h0cfd2490ff577ffca3, + 72'h0bfdc0a0ff706ffce3, + 72'h0afe44c8ff875ffd22, + 72'h09feb300ff9c4ffd62, + 72'h08ff0d40ffaf47fda1, + 72'h07ff5588ffc03ffde1, + 72'h7ff6218013bced8867, + 72'h7fd887882776e588c9, + 72'h7fa736b03b2ac5898b, + 72'h7f6236884ed58d8aaf, + 72'h7f0991c06274258c34, + 72'h7e9d55f07603958e1a, + 72'h7e1d93e08980cd9060, + 72'h7d8a5f389ce8cd9307, + 72'h7ce3cea0b038a5960d, + 72'h7c29fbe0c36d4d9973, + 72'h7b5d0390d683cd9d37, + 72'h7a7d0548e9793da159, + 72'h798a23a0fc4aada5d9, + 72'h788484010ef535aab6, + 72'h776c4ec92175f5afef, + 72'h7641af2933ca0db584, + 72'h7504d33145eeb5bb72, + 72'h73b5ebb157e11dc1bb, + 72'h72552c69699e75c85b, + 72'h70e2cba97b240dcf54, + 72'h6f5f02918c6f2dd6a3, + 72'h6dca0cf19d7d2dde47, + 72'h6c242941ae4b6de63f, + 72'h6a6d9881bed74dee8a, + 72'h68a69e61cf1e45f727, + 72'h66cf8101df1dd60014, + 72'h64e88901eed3860950, + 72'h62f20189fe3ce612da, + 72'h60ec38020d579e1cb0, + 72'h5ed77c621c214626d0, + 72'h5cb420b22a97ae3139, + 72'h5a82797238b88e3bea, + 72'h5842dd224681be46e0, + 72'h55f5a4a253f116521a, + 72'h539b2ac26104865d97, + 72'h5133cc626dba0e6954, + 72'h4ebfe8727a0fbe7550, + 72'h4c3fdfc286039e8188, + 72'h49b415029193e68dfc, + 72'h471cecb29cbec69aa9, + 72'h447acd22a7827ea78c, + 72'h41ce1e32b1dd76b4a5, + 72'h3f174982bbce0ec1f1, + 72'h3c56ba3ac552b6cf6e, + 72'h398cdd02ce69fedd1a, + 72'h36ba1fdad71286eaf3, + 72'h33def252df4ae6f8f7, + 72'h30fbc512e711e70723, + 72'h2e110a2aee664f1576, + 72'h2b1f34b2f5470723ec, + 72'h2826b8f2fbb2ef3285, + 72'h25280c2301a917413d, + 72'h2223a48b07288f5013, + 72'h1f19f93b0c30875f04, + 72'h1c0b822b10c0276e0e, + 72'h18f8b7fb14d6c77d2e, + 72'h15e214031873bf8c62, + 72'h12c810331b96879ba8, + 72'h0fab26eb1e3e9faafe, + 72'h0c8bd323206b9fba61, + 72'h096a900b221d2fc9ce, + 72'h0647d94323530fd944, + 72'h03242a83240d0fe8c0, + 72'h00000003244857f8cf, + 72'h7e8d3859743bdfa2f1, + 72'h7d17824177301fa177, + 72'h7b9ed1d97a307f9ff4, + 72'h7a231ae17d3d3f9e67, + 72'h78a450c98056b79cd1, + 72'h772266c1837d2f9b30, + 72'h759d4f9186b0f79986, + 72'h7414fdc989f26f97d1, + 72'h728963898d41df9610, + 72'h70fa72a1909fb79445, + 72'h6f681c91940c3f926d, + 72'h6dd252519787ef908a, + 72'h6c3904b19b13178e99, + 72'h6a9c23f19eae2f8c9c, + 72'h68fb9fe9a2599f8a91, + 72'h67576811a615cf8878, + 72'h65af6b61a9e33f8651, + 72'h64039871adc25f841b, + 72'h6253dd49b1b3af81d5, + 72'h60a02771b5b7af7f7f, + 72'h5ee86401b9cee77d18, + 72'h5d2c7f79bdf9df7a9f, + 72'h5b6c65c9c239277815, + 72'h59a80259c68d5f7578, + 72'h57df3ff1caf71772c7, + 72'h561208c1cf76f77002, + 72'h54404641d40da76d29, + 72'h5269e151d8bbcf6a39, + 72'h508ec221dd822f6732, + 72'h4eaed011e261876414, + 72'h4cc9f1d1e75a8f60dd, + 72'h4ae00d41ec6e1f5d8c, + 72'h48f10779f19d075a21, + 72'h46fcc4a1f6e82f5699, + 72'h45032811fc507752f4, + 72'h4304143201d6d74f31, + 72'h40ff6a5a077c4f4b4f, + 72'h3ef50b020d41df474b, + 72'h3ce4d5721328a74324, + 72'h3acea7f21931c73ed9, + 72'h38b25f921f5e673a69, + 72'h368fd82225afcf35d0, + 72'h3466ec522c273f310e, + 72'h3237755232c6172c21, + 72'h30014b02398dc72706, + 72'h2dc443da407fcf21bb, + 72'h2b8034b2479dbf1c3f, + 72'h2934f0da4ee937168d, + 72'h26e249e25663ef10a4, + 72'h24880f9a5e0fc70a81, + 72'h2226100265ee970420, + 72'h1fbc17026e0276fd80, + 72'h1d49ee9a764d6ef69b, + 72'h1acf5e827ed1ceef6f, + 72'h184c2c228791e6e7f7, + 72'h15c01a92909036e030, + 72'h132aea4299cf5ed814, + 72'h108c58eaa35226cfa0, + 72'h0de42182ad1b7ec6cd, + 72'h0b31fbe2b72e86bd97, + 72'h08759cbac18e86b3f7, + 72'h05aeb552cc3efea9e8, + 72'h02dcf34ad743a69f61, + 72'h00000002e2a27693f4, + 72'hfd3e0c82be2110f4aa, + 72'hfa83b352b690d8f207, + 72'hf7d0dfeaaf1580ef6c, + 72'hf5257d8aa7aec8ecd8, + 72'hf28177b2a05c78ea4b, + 72'hefe4ba12991e58e7c4, + 72'hed4f309291f430e545, + 72'heac0c75a8addd0e2cd, + 72'he8396ac283daf8e05c, + 72'he5b907527ceb70ddf1, + 72'he33f89e2760f10db8d, + 72'he0ccdf5a6f4598d92f, + 72'hde60f4ea688ed8d6d9, + 72'hdbfbb80261ea98d488, + 72'hd99d162a5b58b0d23e, + 72'hd744fd3254d8e0cffa, + 72'hd4f35b124e6b00cdbd, + 72'hd2a81df2480ed8cb85, + 72'hd063344241c438c954, + 72'hce248c7a3b8af0c729, + 72'hcbec15623562d8c504, + 72'hc9b9bdea2f4bb8c2e4, + 72'hc78d752a294568c0cb, + 72'hc5672a72234fb0beb7, + 72'hc346cd3a1d6a68bca9, + 72'hc12c4d2a179560baa1, + 72'hbf179a1211d070b89f, + 72'hbd08a3fa0c1b68b6a1, + 72'hbaff5b0a067620b4aa, + 72'hb8fbafa200e060b2b8, + 72'hb6fd9239fb5a10b0cb, + 72'hb504f389f5e2f8aee3, + 72'hb311c469f07af8ad01, + 72'hb123f5d9eb21d8ab24, + 72'haf3b7901e5d780a94c, + 72'had583f41e09bb8a779, + 72'hab7a3a09db6e60a5ab, + 72'ha9a15b09d64f50a3e2, + 72'ha7cd9409d13e60a21e, + 72'ha5fed6f9cc3b68a05f, + 72'ha4351601c746409ea5, + 72'ha2704351c25ec89cf0, + 72'ha0b05161bd84d09b3f, + 72'h9ef532b1b8b8409993, + 72'h9d3ed9f1b3f8e897eb, + 72'h9b8d3a01af46a89648, + 72'h99e045d9aaa16094aa, + 72'h9837f099a608e09310, + 72'h96942d81a17d10917a, + 72'h94f4eff19cfdc88fe9, + 72'h935a2b79988ae88e5c, + 72'h91c3d3b99424488cd4, + 72'h9031dc898fc9d08b4f, + 72'h8ea439d18b7b5889cf, + 72'h8d1adfa18738c08853, + 72'h8b95c2298301e886db, + 72'h8a14d5b97ed6b08567, + 72'h88980ec17ab6f883f8, + 72'h871f61d976a2a0828c, + 72'h85aac3a97299808124, + 72'h843a29016e9b887fc0, + 72'h82cd86d96aa8907e5f, + 72'h8164d23166c0807d03, + 72'h8000000162e3687b9e, + 72'h404060a7bf3e78183c, + 72'h40818517be78b818b8, + 72'h40c3713fbdaf081937, + 72'h41062927bce14019b9, + 72'h4149b0e7bc0f481a3f, + 72'h418e0ccfbb39001ac9, + 72'h41d34127ba5e501b57, + 72'h4219528fb97f081be9, + 72'h4260458fb89b101c80, + 72'h42a81ef7b7b2381d1b, + 72'h42f0e3afb6c4701dba, + 72'h433a98c7b5d1801e5e, + 72'h43854377b4d9401f08, + 72'h43d0e917b3db901fb6, + 72'h441d8f3fb2d840206a, + 72'h446b3b97b1cf282124, + 72'h44b9f407b0c01021e4, + 72'h4509beafafaac822a9, + 72'h455aa1cfae8f202375, + 72'h45aca3d7ad6ce82448, + 72'h45ffcb7fac43e02521, + 72'h46541fb7ab13d82602, + 72'h46a9a797a9dc8826ea, + 72'h47006a7fa89dc027da, + 72'h47587017a7573028d2, + 72'h47b1c047a608a029d3, + 72'h480c632fa4b1b82add, + 72'h48686147a352402bf0, + 72'h48c5c347a1e9e02d0d, + 72'h49249247a078502e34, + 72'h4984d79f9efd282f65, + 72'h49e69d179d782030a2, + 72'h4a49ecaf9be8d831ea, + 72'h4aaed0ef9a4ee0333f, + 72'h4b15549f98a9e834a1, + 72'h4b7d830f96f9703610, + 72'h4be767ef953d10378d, + 72'h4c530f579374503919, + 72'h4cc085f7919eb03ab5, + 72'h4d2fd8e78fbbb03c61, + 72'h4da115cf8dcac83e1f, + 72'h4e144adf8bcb603fee, + 72'h4e8986d789bce841d1, + 72'h4f00d937879eb043c8, + 72'h4f7a51f785702045d4, + 72'h4ff601cf83307847f7, + 72'h5073fa4780def84a31, + 72'h50f44d777e7ae84c85, + 72'h51770e7f7c03684ef2, + 72'h51fc512f797798517c, + 72'h52842a4776d6905424, + 72'h530eaf8f741f5056ea, + 72'h539bf7b77150d059d3, + 72'h542c1a8f6e69f05cde, + 72'h54bf31076b6990600f, + 72'h55555537684e706368, + 72'h55eea2a765173066eb, + 72'h568b360f61c2786a9b, + 72'h572b2dbf5e4eb86e7c, + 72'h57cea9775aba60728f, + 72'h5875cab75703b876da, + 72'h5920b4b75328e87b5e, + 72'h59cf8c8f4f28088022, + 72'h5a82796f4afef08528, + 72'h5add84bfa46bd02246, + 72'h5b39a4c7a3543022f5, + 72'h5b96df47a236f023a8, + 72'h5bf539e7a113f02461, + 72'h5c54ba7f9feb00251e, + 72'h5cb567179ebbf825e1, + 72'h5d1745d79d86a826aa, + 72'h5d7a5d179c4ae82779, + 72'h5ddeb3779b0880284e, + 72'h5e444faf99bf382929, + 72'h5eab38a7986ef02a0a, + 72'h5f1375979717582af3, + 72'h5f7d0dd795b8502be2, + 72'h5fe808ff9451882cd9, + 72'h60546edf92e2d02dd8, + 72'h60c24797916be82ede, + 72'h61319b778fec802fed, + 72'h61a2731f8e64603105, + 72'h6214d75f8cd3403225, + 72'h6288d16f8b38d0334f, + 72'h62fe6abf8994c03482, + 72'h6375ad0f87e6c835c0, + 72'h63eea27f862e883709, + 72'h6469557f846bb0385c, + 72'h64e5d0d7829de039bb, + 72'h65641fa780c4b03b26, + 72'h65e44d877edfc83c9e, + 72'h6666665f7ceeb83e23, + 72'h66ea768f7af1183fb6, + 72'h67708aef78e6684157, + 72'h67f8b0b776ce404307, + 72'h6882f5b774a81044c7, + 72'h690f681f7273604697, + 72'h699e16c7702fa04879, + 72'h6a2f10f76ddc404a6d, + 72'h6ac266a76b78a84c75, + 72'h6b5828676904384e90, + 72'h6bf06757667e4050c0, + 72'h6c8b354763e6205306, + 72'h6d28a4df613b105564, + 72'h6dc8c95f5e7c5057da, + 72'h6e6bb6d75ba9185a69, + 72'h6f11823758c0885d14, + 72'h6fba414755c1c05fdb, + 72'h70660aaf52abd062c1, + 72'h7114f62f4f7db065c6, + 72'h71c71c574c366868ec, + 72'h727c96ff48d4c86c36, + 72'h733580ff4557b06fa6, + 72'h73f1f66f41bde8733d, + 72'h74b214af3e061876fe, + 72'h7575fa873a2ed87aeb, + 72'h763dc7ff3636c07f07, + 72'h77099ed7321c308356, + 72'h77d9a2472ddd9087d9, + 72'h78adf74f2979088c95, + 72'h7986c4b724ecd0918c, + 72'h7a6433372036e096c4, + 72'h7b466da71b55109c3f, + 72'h7c2da0ef164528a203, + 72'h7d19fc671104b8a814, + 72'h7e0bb1e70b9128ae78, + 72'h7f02f5df05e7c0b535, + 72'h80000007000438bc93, + 72'h40404047bf7f601018, + 72'h40810207befd201049, + 72'h40c246dfbe7948107a, + 72'h41041047bdf3e810ad, + 72'h41465fe7bd6ce810e0, + 72'h41893757bce4481114, + 72'h41cc982fbc5a081149, + 72'h42108427bbce10117e, + 72'h4254fce7bb407011b5, + 72'h429a042fbab11011ec, + 72'h42df9bb7ba1ff01224, + 72'h4325c547b98d00125e, + 72'h436c82a7b8f8401298, + 72'h43b3d5b7b861b012d3, + 72'h43fbc047b7c938130f, + 72'h44444447b72ed8134c, + 72'h448d63a7b69280138a, + 72'h44d72047b5f43813c9, + 72'h45217c3fb553e8140a, + 72'h456c7987b4b190144b, + 72'h45b81a27b40d28148e, + 72'h4604604fb366a014d1, + 72'h46514e07b2bdf01516, + 72'h469ee587b21318155c, + 72'h46ed2907b1660015a3, + 72'h473c1abfb0b6a015eb, + 72'h478bbcf7b004f81635, + 72'h47dc11f7af51001680, + 72'h482d1c37ae9a9816cc, + 72'h487ede0fade1c8171a, + 72'h48d159e7ad26881769, + 72'h4924924fac68b817b9, + 72'h497889c7aba860180b, + 72'h49cd42e7aae570185f, + 72'h4a22c04faa1fd018b4, + 72'h4a7904afa95780190a, + 72'h4ad012b7a88c701962, + 72'h4b27ed37a7be9819bc, + 72'h4b809707a6ede01a17, + 72'h4bda12ffa61a401a74, + 72'h4c346407a543b01ad3, + 72'h4c8f8d2fa46a181b33, + 72'h4ceb916fa38d701b95, + 72'h4d4873f7a2ada01bfa, + 72'h4da637d7a1caa81c60, + 72'h4e04e057a0e4681cc8, + 72'h4e6470b79ffae01d32, + 72'h4ec4ec4f9f0df81d9e, + 72'h4f2656979e1d981e0c, + 72'h4f88b2f79d29b81e7c, + 72'h4fec04ff9c32481eef, + 72'h505050579b37281f63, + 72'h50b5989f9a38581fda, + 72'h511be1979935c02054, + 72'h51832f27982f4020d0, + 72'h51eb851f9724d8214e, + 72'h5254e78f96166021cf, + 72'h52bf5a879503d02252, + 72'h532ae21f93ed1022d8, + 72'h5397829f92d2082361, + 72'h5405405791b2a023ed, + 72'h54741faf908ec8247c, + 72'h54e425278f6660250d, + 72'h555555578e395825a2, + 72'h55c7b4f78d07882639, + 72'h563b48c78bd0e826d4, + 72'h56b015af8a95582772, + 72'h572620b78954b82814, + 72'h579d6ee7880ef828b9, + 72'h5816058786c3e82962, + 72'h588fe9d78573802a0e, + 72'h590b2167841d902abe, + 72'h5987b1a782c2002b72, + 72'h5a05a0578160b82c2a, + 72'h5a84f3477ff9902ce6, + 72'h5b05b0577e8c682da6, + 72'h5b87ddaf7d19182e6a, + 72'h5c0b816f7b9f802f33, + 72'h5c90a1ff7a1f803000, + 72'h5d1745cf7898f830d2, + 72'h5d9f738f770bb031a9, + 72'h5e2932077577883285, + 72'h5eb4881f73dc603366, + 72'h5f417d07723a00344c, + 72'h5fd017ef7090483538, + 72'h6060605f6edf083629, + 72'h60f25de76d26103720, + 72'h6186185f6b6530381c, + 72'h621b97bf699c38391f, + 72'h62b2e43f67caf83a28, + 72'h634c063765f1403b38, + 72'h63e70637640ed83c4e, + 72'h6483ed276223803d6b, + 72'h6522c3ef602f103e8f, + 72'h65c393df5e31403fba, + 72'h666666675c29e040ed, + 72'h670b45375a18a84228, + 72'h67b23a4f57fd58436a, + 72'h685b4fdf55d7b844b5, + 72'h6906906753a7784608, + 72'h69b40697516c584765, + 72'h6a63bd774f261048ca, + 72'h6b15c0674cd4584a38, + 72'h6bca1ae74a76d84bb1, + 72'h6c80d8f7480d484d33, + 72'h6d3a06cf4597504ebf, + 72'h6df5b0e74314a85057, + 72'h6eb3e4474084e851f9, + 72'h6f74ae173de7b853a7, + 72'h70381c073b3cc05560, + 72'h70fe3bf73883985726, + 72'h71c71c6735bbd858f8, + 72'h7292cc0732e5285ad8, + 72'h73615a172fff085cc5, + 72'h7432d6372d09105ec0, + 72'h750750672a02d060ca, + 72'h75ded94726ebc862e2, + 72'h76b981c723c380650b, + 72'h77975b7f2089706743, + 72'h787878671d3d18698c, + 72'h795ceb0f19ddf06be7, + 72'h7a44c69f166b586e54, + 72'h7b301eb712e4d070d3, + 72'h7c1f07af0f49a87366, + 72'h7d1196670b9950760d, + 72'h7e07e06707d31078c9, + 72'h7f01fbef03f6487b9b, + 72'h800000070001b07e9f + }; + + reg [8:0] base; + + always @(*) begin + case (tbl) + 3'd0: base = 9'd0; + 3'd1: base = 9'd128; + 3'd2: base = 9'd256; + 3'd3: base = 9'd320; + 3'd4: base = 9'd384; + 3'd5: base = 9'd448; + default: base = '0; + endcase + end + + wire [8:0] addr = base + 9'(idx); + + wire [DATAW-1:0] rd_data, rd_data_r; + + VX_rom #( + .DATAW (DATAW), + .SIZE (SIZE), + .OUT_REG (1), + .INIT_DATA (ROM_DATA) + ) rom ( + .clk (clk), + .read (enable), + .addr (addr), + .rdata (rd_data) + ); + + VX_pipe_register #( + .DATAW (DATAW), + .DEPTH (1) + ) rd_reg ( + .clk (clk), + .reset (1'b0), + .enable (enable), + .data_in (rd_data), + .data_out (rd_data_r) + ); + + assign a0 = 32'({rd_data_r[71:43], 2'b0}); + assign a1 = 27'($signed({rd_data_r[42:19], 2'b0})); + assign a2 = 20'($signed({rd_data_r[18:0], 1'b0})); + +endmodule diff --git a/hw/rtl/mufu/VX_mufu_unit.sv b/hw/rtl/mufu/VX_mufu_unit.sv new file mode 100644 index 0000000000..93109a1787 --- /dev/null +++ b/hw/rtl/mufu/VX_mufu_unit.sv @@ -0,0 +1,93 @@ +`include "VX_define.vh" + +module VX_mufu_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1 +) ( + input wire clk, + input wire reset, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam NUM_PES = `VX_CFG_NUM_MUFU_LANES; + localparam LATENCY = `VX_MUFU_LATENCY; + localparam HDR_WIDTH = $bits(sfu_header_t); + + `STATIC_ASSERT ((NUM_LANES % NUM_PES) == 0, ("invalid parameter: NUM_LANES=%0d, NUM_MUFU_LANES=%0d", NUM_LANES, NUM_PES)) + + `UNUSED_VAR (execute_if.data.op_type) + `UNUSED_VAR (execute_if.data.op_args) + `UNUSED_VAR (execute_if.data.rs2_data) + `UNUSED_VAR (execute_if.data.rs3_data) + + wire [NUM_LANES-1:0][31:0] req_data; + wire [NUM_LANES-1:0][31:0] rsp_data; + wire [NUM_PES-1:0][31:0] pe_data_in; + wire [NUM_PES-1:0][31:0] pe_data_out; + wire [INST_MUFU_BITS-1:0] pe_fn; + wire pe_enable; + sfu_header_t rsp_hdr; + + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_req_data + wire [`VX_CFG_XLEN-1:0] rs1 = execute_if.data.rs1_data[i]; + if (`VX_CFG_XLEN > 32) begin : g_unbox + assign req_data[i] = (&rs1[`VX_CFG_XLEN-1:32]) ? rs1[31:0] : 32'h7fc00000; + end else begin : g_nobox + assign req_data[i] = rs1[31:0]; + end + end + + VX_pe_serializer #( + .NUM_LANES (NUM_LANES), + .NUM_PES (NUM_PES), + .LATENCY (LATENCY), + .DATA_IN_WIDTH (32), + .DATA_OUT_WIDTH (32), + .SHARED_WIDTH (INST_MUFU_BITS), + .TAG_WIDTH (HDR_WIDTH), + .PE_REG (0), + .OUT_BUF (0) + ) pe_serializer ( + .clk (clk), + .reset (reset), + .valid_in (execute_if.valid), + .mask_in (execute_if.data.header.tmask), + .data_in (req_data), + .shared_in (execute_if.data.op_args.mufu.fn), + .tag_in (execute_if.data.header), + .ready_in (execute_if.ready), + .pe_enable (pe_enable), + `UNUSED_PIN (pe_mask_out), + .pe_data_out (pe_data_in), + .pe_shared_out (pe_fn), + .pe_data_in (pe_data_out), + .valid_out (result_if.valid), + `UNUSED_PIN (mask_out), + .data_out (rsp_data), + .tag_out (rsp_hdr), + .ready_out (result_if.ready) + ); + + for (genvar i = 0; i < NUM_PES; ++i) begin : g_pes + VX_mufu_pe pe ( + .clk (clk), + .reset (reset), + .enable (pe_enable), + .fn (pe_fn), + .dataa (pe_data_in[i]), + .result (pe_data_out[i]) + ); + end + + assign result_if.data.header = rsp_hdr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data + if (`VX_CFG_XLEN > 32) begin : g_box + assign result_if.data.data[i] = {{(`VX_CFG_XLEN-32){1'b1}}, rsp_data[i]}; + end else begin : g_nobox + assign result_if.data.data[i] = rsp_data[i]; + end + end + +endmodule diff --git a/hw/scripts/gen_mufu_tables.py b/hw/scripts/gen_mufu_tables.py new file mode 100644 index 0000000000..0011b85a11 --- /dev/null +++ b/hw/scripts/gen_mufu_tables.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the MUFU coefficient ROM. + +Every table interval is approximated as P = A0 + A1*z + A2*z^2 in 30-bit fixed +point, z = xl/2^18 being the position inside the interval. Evaluation is the +bit-exact integer form shared by sw/common/mufu.cpp and hw/rtl/mufu: + + P = A0 + ((A1*xl) >> 18) + ((A2*((xl*xl) >> 18)) >> 18) + +The ROM stores each coefficient without its S0/S1/S2 low bits (Ak = stored << +Sk), so one entry is 72 bits and the 512 entries fill one 512x72 RAMB36. + +Outputs (checked in, regenerate after any change here): + sw/common/mufu_tables.h + hw/rtl/mufu/VX_mufu_rom.sv + +Usage: + gen_mufu_tables.py write both outputs + gen_mufu_tables.py --check fail if the checked-in outputs are stale + gen_mufu_tables.py --report print per-table interpolation error +""" + +import argparse +import math +import os +import sys + +import numpy as np + +FRAC = 30 +XL_BITS = 18 +# Coefficient widths seen by the datapath, and low bits dropped in the ROM. +A0_BITS = 32 +A1_BITS = 27 +A2_BITS = 20 +S0, S1, S2 = 2, 2, 1 +W0, W1, W2 = A0_BITS - 1 - S0, A1_BITS - 1 - S1, A2_BITS - S2 +ROM_DEPTH = 512 + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +OUT_H = os.path.join(ROOT, "sw", "common", "mufu_tables.h") +OUT_SV = os.path.join(ROOT, "hw", "rtl", "mufu", "VX_mufu_rom.sv") + + +# Intervals per table, sized for PTX .approx-level accuracy; --report prints the +# resulting worst-case errors. +N_RCP = 128 # 1/(1+t) +N_RSQRT = 64 # per exponent parity +N_EX2 = 64 +N_LG2 = 64 +N_TRIG = 64 # sin(pi/2 t); cos reads it at 1 - t +TANH_J0 = -5 # first tanh binade (below it tanh x = x) +TANH_BINADES = 8 # binades 2^-5 .. 2^2 (from 8 up tanh x = 1) +TANH_PER = 8 # intervals per tanh binade +TANH_PRESCALE = 4 # tanh entries hold 2^-(j+4)*tanh, keeping A1 in range + + +def _rcp(idx, z): + return 1.0 / (1.0 + (idx + z) / N_RCP) + + +def _rsqrt(idx, z): + t = ((idx % N_RSQRT) + z) / N_RSQRT + s = (1.0 + t) * (2.0 if (idx // N_RSQRT) else 1.0) + return 1.0 / np.sqrt(s) + + +def _ex2(idx, z): + return np.exp2((idx + z) / N_EX2) + + +def _lg2(idx, z): + return np.log2(1.0 + (idx + z) / N_LG2) + + +def _sin(idx, z): + return np.sin(np.pi / 2.0 * (idx + z) / N_TRIG) + + +def _tanh(idx, z): + j = (idx // TANH_PER) + TANH_J0 + t = ((idx % TANH_PER) + z) / TANH_PER + return np.ldexp(np.tanh(np.ldexp(1.0 + t, j)), -j - TANH_PRESCALE) + + +def _mant_step(n): + # xl takes the mantissa bits below the index, left-aligned in XL_BITS. + low = 23 - (n.bit_length() - 1) + return (1 << (XL_BITS - low)) if low < XL_BITS else 1, low > XL_BITS + + +# name, entries, f(idx, z), xl step (xl values are multiples of it), +# xl truncated (true argument lies anywhere in [xl, xl+1)), exact-left intervals +TABLES = [ + ("RCP", N_RCP, _rcp, *_mant_step(N_RCP), {0}), + ("RSQRT", 2 * N_RSQRT, _rsqrt, *_mant_step(N_RSQRT), {0}), + ("EX2", N_EX2, _ex2, 1, True, {0}), + ("LG2", N_LG2, _lg2, *_mant_step(N_LG2), {0}), + ("SIN", N_TRIG, _sin, 1, True, {0}), + ("TANH", TANH_BINADES * TANH_PER, _tanh, *_mant_step(TANH_PER), set()), +] + + +def eval_fixed(a0, a1, a2, xl): + sq = (xl * xl) >> XL_BITS + return a0 + ((a1 * xl) >> XL_BITS) + ((a2 * sq) >> XL_BITS) + + +def minimax(A, y, iters=40): + # Lawson's iteration: reweighted least squares converging to the minimax fit. + w = np.full(len(y), 1.0 / len(y)) + sol = None + for _ in range(iters): + sw = np.sqrt(w) + sol, *_ = np.linalg.lstsq(A * sw[:, None], y * sw, rcond=None) + w = w * (np.abs(A @ sol - y) + 1e-12) + w /= w.sum() + return sol + + +def fit_interval(f, idx, step, trunc, exact_left): + off = 0.5 if trunc else 0.0 + zs = (np.linspace(0, (1 << XL_BITS) - step, 2049) + off) / float(1 << XL_BITS) + ys = f(idx, zs) * (1 << FRAC) + + if exact_left: + a0 = int(round(float(f(idx, 0.0)) * (1 << FRAC))) + a1, a2 = minimax(np.stack([zs, zs * zs], axis=1), ys - a0) + else: + a0, a1, a2 = minimax(np.stack([np.ones_like(zs), zs, zs * zs], axis=1), ys) + a0 = int(round(a0)) + a1 = int(round(a1 / (1 << S1))) << S1 + a2 = int(round(a2 / (1 << S2))) << S2 + + xl = np.arange(0, 1 << XL_BITS, step, dtype=np.int64) + target = f(idx, (xl + off) / float(1 << XL_BITS)) * (1 << FRAC) + if not exact_left: + err = eval_fixed(a0, a1, a2, xl) - target + a0 -= int(round((err.max() + err.min()) / 2.0)) + assert not exact_left or a0 % (1 << S0) == 0, (idx, a0) + a0 = int(round(a0 / (1 << S0))) << S0 + err = eval_fixed(a0, a1, a2, xl) - target + return a0, a1, a2, float(np.abs(err).max()) + + +def build(): + entries = [] + report = [] + bases = [] + base = 0 + for name, n, f, step, trunc, exact in TABLES: + bases.append((name, base, n)) + worst = 0.0 + wid = [0, 0, 0] + for idx in range(n): + a0, a1, a2, e = fit_interval(f, idx, step, trunc, idx in exact) + q0, q1, q2 = a0 >> S0, a1 >> S1, a2 >> S2 + assert 0 <= q0 < (1 << W0), (name, idx, a0) + assert -(1 << (W1 - 1)) <= q1 < (1 << (W1 - 1)), (name, idx, a1) + assert -(1 << (W2 - 1)) <= q2 < (1 << (W2 - 1)), (name, idx, a2) + entries.append((q0, q1, q2)) + worst = max(worst, e) + wid = [max(wid[0], a0.bit_length()), max(wid[1], abs(a1).bit_length() + 1), + max(wid[2], abs(a2).bit_length() + 1)] + report.append((name, worst, n, wid)) + base += n + assert len(entries) <= ROM_DEPTH and W0 + W1 + W2 <= 72 + return entries, bases, report + + +def emit_h(entries, bases): + out = [] + out.append("// Generated by hw/scripts/gen_mufu_tables.py. Do not edit.") + out.append("#pragma once") + out.append("") + out.append("#include ") + out.append("") + out.append("namespace vortex::mufu {") + out.append("") + out.append("// Stored coefficients; the datapath uses a0 << S0, a1 << S1, a2 << S2.") + out.append("struct coeff_t { uint32_t a0; int32_t a1; int32_t a2; };") + out.append("") + out.append(f"constexpr uint32_t S0 = {S0};") + out.append(f"constexpr uint32_t S1 = {S1};") + out.append(f"constexpr uint32_t S2 = {S2};") + out.append("") + for name, base, n in bases: + out.append(f"constexpr uint32_t TBL_{name}_BASE = {base};") + out.append(f"constexpr uint32_t TBL_SIZE = {len(entries)};") + out.append("") + out.append("constexpr coeff_t kTable[TBL_SIZE] = {") + for a0, a1, a2 in entries: + out.append(f" {{0x{a0:08x}u, {a1}, {a2}}},") + out.append("};") + out.append("") + out.append("} // namespace vortex::mufu") + out.append("") + return "\n".join(out) + + +def emit_sv(entries, bases): + aw = (ROM_DEPTH - 1).bit_length() + iw = max(1, (max(n for _, _, n in bases) - 1).bit_length()) + tw = max(1, (len(bases) - 1).bit_length()) + w = W0 + W1 + W2 + words = [] + for i in range(ROM_DEPTH): + q0, q1, q2 = entries[i] if i < len(entries) else (0, 0, 0) + words.append((q0 << (W1 + W2)) | ((q1 & ((1 << W1) - 1)) << W2) | (q2 & ((1 << W2) - 1))) + out = [] + out.append("`include \"VX_platform.vh\"") + out.append("") + out.append("module VX_mufu_rom (") + out.append(" input wire clk,") + out.append(" input wire enable,") + out.append(f" input wire [{tw - 1}:0] tbl,") + out.append(f" input wire [{iw - 1}:0] idx,") + out.append(f" output wire [{A0_BITS - 1}:0] a0,") + out.append(f" output wire signed [{A1_BITS - 1}:0] a1,") + out.append(f" output wire signed [{A2_BITS - 1}:0] a2") + out.append(");") + out.append(f" localparam DATAW = {w};") + out.append(f" localparam SIZE = {ROM_DEPTH};") + out.append("") + out.append(" localparam [SIZE*DATAW-1:0] ROM_DATA = {") + for i in range(ROM_DEPTH - 1, -1, -1): + sep = "," if i > 0 else "" + out.append(f" {w}'h{words[i]:0{(w + 3) // 4}x}{sep}") + out.append(" };") + out.append("") + out.append(f" reg [{aw - 1}:0] base;") + out.append("") + out.append(" always @(*) begin") + out.append(" case (tbl)") + for i, (_, base, _) in enumerate(bases): + out.append(f" {tw}'d{i}: base = {aw}'d{base};") + out.append(f" default: base = '0;") + out.append(" endcase") + out.append(" end") + out.append("") + out.append(f" wire [{aw - 1}:0] addr = base + {aw}'(idx);") + out.append("") + out.append(" wire [DATAW-1:0] rd_data, rd_data_r;") + out.append("") + out.append(" VX_rom #(") + out.append(" .DATAW (DATAW),") + out.append(" .SIZE (SIZE),") + out.append(" .OUT_REG (1),") + out.append(" .INIT_DATA (ROM_DATA)") + out.append(" ) rom (") + out.append(" .clk (clk),") + out.append(" .read (enable),") + out.append(" .addr (addr),") + out.append(" .rdata (rd_data)") + out.append(" );") + out.append("") + out.append(" VX_pipe_register #(") + out.append(" .DATAW (DATAW),") + out.append(" .DEPTH (1)") + out.append(" ) rd_reg (") + out.append(" .clk (clk),") + out.append(" .reset (1'b0),") + out.append(" .enable (enable),") + out.append(" .data_in (rd_data),") + out.append(" .data_out (rd_data_r)") + out.append(" );") + out.append("") + out.append(f" assign a0 = {A0_BITS}'({{rd_data_r[{w - 1}:{W1 + W2}], {S0}'b0}});") + out.append(f" assign a1 = {A1_BITS}'($signed({{rd_data_r[{W1 + W2 - 1}:{W2}], {S1}'b0}}));") + out.append(f" assign a2 = {A2_BITS}'($signed({{rd_data_r[{W2 - 1}:0], {S2}'b0}}));") + out.append("") + out.append("endmodule") + out.append("") + return "\n".join(out) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--check", action="store_true") + ap.add_argument("--report", action="store_true") + args = ap.parse_args() + + entries, bases, report = build() + h = emit_h(entries, bases) + sv = emit_sv(entries, bases) + + if args.report: + for name, worst, n, wid in report: + print(f"{name:6s} {n:4d} entries max interpolation error {worst:8.2f} units of 2^-{FRAC}" + f" (2^{math.log2(max(worst, 1e-9) / (1 << FRAC)):.2f}) bits a0/a1/a2 = {wid[0]}/{wid[1]}/{wid[2]}") + print(f"total {len(entries)} entries") + + if args.check: + stale = [] + for path, text in ((OUT_H, h), (OUT_SV, sv)): + try: + with open(path) as fp: + if fp.read() != text: + stale.append(path) + except FileNotFoundError: + stale.append(path) + if stale: + print("stale: " + ", ".join(stale), file=sys.stderr) + return 1 + return 0 + + if not args.report: + os.makedirs(os.path.dirname(OUT_SV), exist_ok=True) + with open(OUT_H, "w") as fp: + fp.write(h) + with open(OUT_SV, "w") as fp: + fp.write(sv) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/syn/extensions.mk b/hw/syn/extensions.mk index 8b549309f5..5d71680331 100644 --- a/hw/syn/extensions.mk +++ b/hw/syn/extensions.mk @@ -90,3 +90,8 @@ ifneq (,$(filter -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_EXT_OM_ENABLE=1 -DVX_CFG_EXT_OM RTL_PKGS += $(RTL_DIR)/om/VX_om_pkg.sv RTL_INCLUDE += -I$(RTL_DIR)/om endif + +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE -DVX_CFG_EXT_MUFU_ENABLE=1 -DVX_CFG_EXT_MUFU_ENABLED=1, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 1bb5edaa98..9a2cde4690 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -10,7 +10,7 @@ # Add a DUT: append its name to DUTS and define the five variables. No new # directory, no dispatcher edit. -DUTS := cache core cp dxa fpu gfx issue lmem mem_unit om raster rtu \ +DUTS := cache core cp dxa fpu gfx issue lmem mem_unit mufu om raster rtu \ scope tcu tensor tex top unittest vm vortex # --------------------------------------------------------------------------- @@ -61,6 +61,13 @@ dxa_PKG = $(RTL_DIR)/dxa/VX_dxa_pkg.sv fpu_IP := 1 fpu_PRJ = $(if $(filter -DVX_CFG_FPU_TYPE_FPNEW,$(XCONFIGS)),VX_fpu_fpnew,$(if $(filter -DVX_CFG_FPU_TYPE_STD,$(XCONFIGS)),VX_fpu_std,VX_fpu_dsp)) fpu_INC = $(BASE_INC) $(FPU_INC) +# BASE_INC pulls in hw/rtl/VX_cluster.sv, which imports VX_tlb_pkg. +fpu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv + +mufu_PRJ := VX_mufu_unit_top +mufu_CFG := -DVX_CFG_EXT_MUFU_ENABLE +mufu_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) -I$(RTL_DIR)/mufu -I$(UNITTEST_DIR)/mufu_unit +mufu_PKG = $(RTL_DIR)/vm/VX_tlb_pkg.sv gfx_PRJ := VX_gfx_top gfx_IP := 1 diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index d67a94c75b..0403566afe 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -9,7 +9,7 @@ # synthesizes, so both gates measure the same modules and a divergence between # them is meaningful rather than an artefact of two DUT lists. -DUTS := cache core tcu gfx tex raster om rtu dxa vm tensor top vortex +DUTS := cache core tcu gfx tex raster om rtu dxa mufu vm tensor top vortex UNITTEST_DIR ?= $(VORTEX_HOME)/hw/unittest AFU_RTL_DIR ?= $(VORTEX_HOME)/hw/rtl/afu @@ -60,6 +60,10 @@ dxa_TOP := VX_dxa_core_top dxa_INC := -I$(UNITTEST_DIR)/dxa_core dxa_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_DXA_ENABLE +mufu_TOP := VX_mufu_unit_top +mufu_INC := -I$(UNITTEST_DIR)/mufu_unit +mufu_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_MUFU_ENABLE + vm_TOP := VX_vm_top vm_INC := -I$(UNITTEST_DIR)/vm # Mirrors hw/syn/xilinx/dut/catalog.mk's vm_CFG. L2_ENABLE is load-bearing, not diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 0387f2a267..001ab90c2a 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -28,6 +28,7 @@ all: $(MAKE) -C fcvt_unit $(MAKE) -C fdivsqrt_unit $(MAKE) -C tex_lerp + $(MAKE) -C mufu_unit run: $(MAKE) -C generic_queue run @@ -59,6 +60,7 @@ run: $(MAKE) -C fcvt_unit run $(MAKE) -C fdivsqrt_unit run $(MAKE) -C tex_lerp run + $(MAKE) -C mufu_unit run # FPU arithmetic units only — executed (not just built) so CI gates correctness. run-fpu: @@ -68,6 +70,10 @@ run-fpu: $(MAKE) -C fcvt_unit run $(MAKE) -C fdivsqrt_unit run +# MUFU datapath, bit-exact against the sw/common golden model. +run-mufu: + $(MAKE) -C mufu_unit run + # TCU FEDP datapath — executed per format so a tcu/tfr change is gated by CI. # Each format must enable its own RTL datapath (FP16-only is the default). Covers # the native TFR and the DPI reference. Float formats compare against the windowed @@ -154,3 +160,4 @@ clean: $(MAKE) -C fcvt_unit clean $(MAKE) -C fdivsqrt_unit clean $(MAKE) -C tex_lerp clean + $(MAKE) -C mufu_unit clean diff --git a/hw/unittest/fcvt_unit/Makefile b/hw/unittest/fcvt_unit/Makefile index 7e5bf4d7d7..fae0143d34 100644 --- a/hw/unittest/fcvt_unit/Makefile +++ b/hw/unittest/fcvt_unit/Makefile @@ -24,6 +24,12 @@ PARAMS += -GSUBNORM_ENABLE=0 -GEXCEPT_ENABLE=0 CXXFLAGS += -DLEAN endif +HALF ?= 1 +ifeq ($(HALF),1) +PARAMS += -GHALF_ENABLE=1 +CXXFLAGS += -DHALF +endif + SRCS += $(SW_COMMON_DIR)/softfloat_ext.cpp SRCS += $(SW_COMMON_DIR)/rvfloats.cpp SRCS += $(SRC_DIR)/main.cpp diff --git a/hw/unittest/fcvt_unit/main.cpp b/hw/unittest/fcvt_unit/main.cpp index 1a7e5074ee..f81707deaa 100644 --- a/hw/unittest/fcvt_unit/main.cpp +++ b/hw/unittest/fcvt_unit/main.cpp @@ -12,8 +12,9 @@ // limitations under the License. // Functional test for VX_fcvt_unit (XLEN=32 / F32): exercises I2F and F2I in -// both signed and unsigned forms and checks bit-exact result/fflags against the -// softfloat (rvfloats) RISC-V golden across all rounding modes. Strict check. +// both signed and unsigned forms, plus F32<->F16 with HALF, and checks +// bit-exact result/fflags against the softfloat (rvfloats) RISC-V golden across +// all rounding modes. Strict check. #include "VVX_fcvt_unit.h" #include "vl_simulator.h" @@ -32,11 +33,15 @@ using Device = VVX_fcvt_unit; -enum { I2F_S, I2F_U, F2I_S, F2I_U, NUM_OPS }; +#ifdef HALF +enum { I2F_S, I2F_U, F2I_S, F2I_U, F2H, H2F, NUM_OPS }; +#else +enum { I2F_S, I2F_U, F2I_S, F2I_U, NUM_OPS, F2H, H2F }; +#endif namespace { -const char* op_name[NUM_OPS] = {"i2f_s","i2f_u","f2i_s","f2i_u"}; +const char* op_name[] = {"i2f_s","i2f_u","f2i_s","f2i_u","f2h","h2f"}; struct TV { uint32_t a, frm; int op; }; struct Golden { uint32_t res, fflags, a, frm; int op; }; @@ -48,8 +53,16 @@ const uint32_t kSpecials[] = { 0x7f800000, 0xff800000, 0x7fc00000, 0x4f000000, 0xcf000000, 0x4effffff, 0x00000001, 0xffffffff, 0x7fffffff, 0x00000002, 0x000003e8, 0x40490fdb, 0x5f000000, 0x5effffff, 0x3f000000, 0x4c3b0244, + 0x477fe000, 0x477fefff, 0x477ff000, 0x477ff001, 0x47800000, 0xc77ff000, + 0x38800000, 0x387fffff, 0x387ff000, 0x387fe000, 0x33800000, 0x33000000, + 0x33000001, 0x32ffffff, 0x3f801000, 0x3f803000, 0x7f800001, 0x7f7fffff, + 0xffff7c00, 0xffff7c01, 0xffff7e00, 0xffff0001, 0xffff83ff, 0xffff3c00, + 0x00003c00, 0x7fff7bff, }; +inline uint32_t flush_sub(uint32_t x){ return (((x>>23)&0xFF)==0 && (x&0x7FFFFF)) ? (x&0x80000000u) : x; } +inline uint16_t flush_sub16(uint16_t x){ return (((x>>10)&0x1F)==0 && (x&0x3FF)) ? (x&0x8000u) : x; } + uint32_t golden(int op, uint32_t a, uint32_t frm, uint32_t* ff) { *ff = 0; switch (op) { @@ -57,6 +70,8 @@ uint32_t golden(int op, uint32_t a, uint32_t frm, uint32_t* ff) { case I2F_U: return rv_utof_s(a, frm, ff); case F2I_S: return rv_ftoi_s(a, frm, ff); case F2I_U: return rv_ftou_s(a, frm, ff); + case F2H: return 0xffff0000u | rv_ftoh_s(a, frm, ff); + case H2F: return ((a >> 16) == 0xffff) ? rv_htof_s(uint16_t(a), frm, ff) : 0x7fc00000u; } return 0; } @@ -65,17 +80,19 @@ void drive(Device& dut, int op, uint32_t a, uint32_t frm) { dut.frm = frm; dut.is_itof = (op == I2F_S || op == I2F_U); dut.is_ftoi = (op == F2I_S || op == F2I_U); - dut.is_f2f = 0; + dut.is_f2f = (op == F2H || op == H2F); dut.is_signed = (op == I2F_S || op == F2I_S); dut.is_int64 = 0; // XLEN=32 dut.src_fmt = 0; // F32 dut.dst_fmt = 0; // F32 +#ifdef HALF + dut.src_h = (op == H2F); + dut.dst_h = (op == F2H); +#endif dut.dataa = a; dut.mask = 1; } -inline uint32_t flush_sub(uint32_t x){ return (((x>>23)&0xFF)==0 && (x&0x7FFFFF)) ? (x&0x80000000u) : x; } - } // namespace int main(int argc, char** argv) { @@ -93,8 +110,16 @@ int main(int argc, char** argv) { for (uint32_t a : kSpecials) for (uint32_t f = 0; f < 5; ++f) tests.push_back({a, f, op}); - for (size_t i = 0; i < num_random; ++i) - tests.push_back({static_cast(rng()), static_cast(i % 5), op}); + for (size_t i = 0; i < num_random; ++i) { + uint32_t a = static_cast(rng()); + if (op == H2F && (i % 16) != 0) { + a |= 0xffff0000u; + } + if (op == F2H && (i % 2) == 0) { + a = (a & 0x807fffffu) | ((97u + (a >> 24) % 50u) << 23); + } + tests.push_back({a, static_cast(i % 5), op}); + } } const size_t N = tests.size(); @@ -117,7 +142,12 @@ int main(int argc, char** argv) { uint32_t ff = 0; uint32_t ga = t.a; #ifdef LEAN - if (t.op == F2I_S || t.op == F2I_U) ga = flush_sub(t.a); // DAZ float source + if (t.op == F2I_S || t.op == F2I_U || t.op == F2H) { + ga = flush_sub(t.a); // DAZ float source + } + if (t.op == H2F && (t.a >> 16) == 0xffff) { + ga = 0xffff0000u | flush_sub16(uint16_t(t.a)); + } #endif uint32_t r = golden(t.op, ga, t.frm, &ff); #ifdef LEAN @@ -146,7 +176,9 @@ int main(int argc, char** argv) { } printf("fcvt_unit: checked=%zu errors=%zu per-op:", checked, errors); - for (int op = 0; op < NUM_OPS; ++op) printf(" %s=%zu", op_name[op], per_op[op]); + for (int op = 0; op < (int)NUM_OPS; ++op) { + printf(" %s=%zu", op_name[op], per_op[op]); + } printf("\n%s\n", errors ? "FAILED" : "PASSED"); return errors ? 1 : 0; } diff --git a/hw/unittest/mufu_unit/Makefile b/hw/unittest/mufu_unit/Makefile new file mode 100644 index 0000000000..cffc5efa2f --- /dev/null +++ b/hw/unittest/mufu_unit/Makefile @@ -0,0 +1,26 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mufu_unit + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +CONFIGS += -DSIMULATION -DVX_CFG_EXT_MUFU_ENABLE + +SRCS += $(SW_COMMON_DIR)/mufu.cpp +SRCS += $(SRC_DIR)/main.cpp + +RTL_PKGS += $(RTL_DIR)/VX_gpu_pkg.sv +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(RTL_DIR) -I$(RTL_DIR)/libs +RTL_INCLUDE += -I$(RTL_DIR)/mufu + +# Generated headers (VX_config.vh, VX_types.vh) live in the build tree; add them +# as an include-search path only, not a source-discovery root. +VL_FLAGS += -I$(ROOT_DIR)/hw + +TOP := VX_mufu_pe + +include ../common.mk diff --git a/hw/unittest/mufu_unit/VX_mufu_unit_top.sv b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv new file mode 100644 index 0000000000..0097f58929 --- /dev/null +++ b/hw/unittest/mufu_unit/VX_mufu_unit_top.sv @@ -0,0 +1,41 @@ +`include "VX_define.vh" + +module VX_mufu_unit_top import VX_gpu_pkg::*; ( + input wire clk, + input wire reset, + + input wire execute_valid, + output wire execute_ready, + input wire [$bits(sfu_execute_t)-1:0] execute_data, + + output wire result_valid, + input wire result_ready, + output wire [$bits(sfu_result_t)-1:0] result_data +); + VX_execute_if #( + .data_t (sfu_execute_t) + ) execute_if(); + + VX_result_if #( + .data_t (sfu_result_t) + ) result_if(); + + assign execute_if.valid = execute_valid; + assign execute_if.data = execute_data; + assign execute_ready = execute_if.ready; + + assign result_valid = result_if.valid; + assign result_data = result_if.data; + assign result_if.ready = result_ready; + + VX_mufu_unit #( + .INSTANCE_ID ("mufu_unit_top"), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES) + ) mufu_unit ( + .clk (clk), + .reset (reset), + .execute_if (execute_if), + .result_if (result_if) + ); + +endmodule diff --git a/hw/unittest/mufu_unit/main.cpp b/hw/unittest/mufu_unit/main.cpp new file mode 100644 index 0000000000..d9045aa490 --- /dev/null +++ b/hw/unittest/mufu_unit/main.cpp @@ -0,0 +1,123 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Bit-exact check of the VX_mufu_pe pipeline against the shared golden model +// (sw/common/mufu.cpp) for every function: special values, a strided sweep of +// the whole FP32 space, and random inputs. +// +// usage: mufu_unit [stride] [num_random] + +#include "VVX_mufu_pe.h" +#include "vl_simulator.h" +#include "mufu.h" + +#include +#include +#include +#include +#include +#include + +using Device = VVX_mufu_pe; +using namespace vortex::mufu; + +namespace { + +constexpr int kLatency = 7; + +const uint32_t kSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, + 0x3f800000, 0xbf800000, 0x3f000000, 0x3fc00000, 0x40000000, 0x40800000, + 0x40490fdb, 0x3eaaaaab, 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, + 0x7fc00000, 0xffc00000, 0x7fa00000, 0x42fe0000, 0x43000000, 0xc2fc0000, + 0xc2fe0000, 0xc3000000, 0x4b000001, 0x4b800001, 0x3f7fffff, 0x3f800001, + 0x39800000, 0x39000000, 0x41000000, 0x40ffffff, 0x3a7fffff, 0x3a800000, +}; + +struct Req { uint32_t fn, x; }; + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + + uint64_t stride = 65521; + size_t num_random = 200000; + if (argc > 1) { + stride = std::strtoull(argv[1], nullptr, 0); + } + if (argc > 2) { + num_random = std::strtoull(argv[2], nullptr, 0); + } + if (stride == 0) { + stride = 1; + } + + std::vector reqs; + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { + for (uint32_t x : kSpecials) { + reqs.push_back({fn, x}); + } + for (uint64_t x = 0; x < (1ull << 32); x += stride) { + reqs.push_back({fn, uint32_t(x)}); + } + } + std::mt19937 rng(0x6d756675u); + for (size_t i = 0; i < num_random; ++i) { + reqs.push_back({uint32_t(i % FN_COUNT), uint32_t(rng())}); + } + + vl_simulator sim; + uint64_t ticks = 0; + sim->enable = 1; + sim->fn = 0; + sim->dataa = 0; + ticks = sim.reset(ticks); + + std::deque inflight; + size_t feed = 0, checked = 0; + size_t errors[FN_COUNT] = {}; + size_t total_errors = 0; + + for (size_t c = 0; c < reqs.size() + kLatency; ++c) { + if (feed < reqs.size()) { + sim->fn = reqs[feed].fn; + sim->dataa = reqs[feed].x; + inflight.push_back(reqs[feed]); + ++feed; + } + ticks = sim.step(ticks, 2); + + if (c >= size_t(kLatency - 1) && !inflight.empty()) { + Req r = inflight.front(); + inflight.pop_front(); + uint32_t got = sim->result; + uint32_t exp = eval(r.fn, r.x); + if (got != exp) { + if (total_errors < 20) { + printf("MISMATCH %s x=%08x : got %08x expected %08x\n", fn_name(r.fn), r.x, got, exp); + } + ++errors[r.fn]; + ++total_errors; + } + ++checked; + } + } + + printf("mufu_unit: checked=%zu errors=%zu [", checked, total_errors); + for (uint32_t fn = 0; fn < FN_COUNT; ++fn) { + printf("%s%s=%zu", fn ? " " : "", fn_name(fn), errors[fn]); + } + printf("]\n%s\n", total_errors ? "FAILED" : "PASSED"); + return total_errors ? 1 : 0; +} diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index dfe8ab9bb4..863442440d 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -133,6 +133,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index f4dbfb723a..55d1d9d214 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -130,6 +130,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index f15c7da8a7..ed85464742 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -122,6 +122,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SIM_COMMON_DIR)/dram_sim.cpp $(SIM_COMMON_DIR)/elf_loader.cpp $(SIM_COMMON_DIR)/host_monitor.cpp $(SIM_COMMON_DIR)/cout_drainer.cpp diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 746bd6f7f8..4338c8ab6d 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -55,6 +55,12 @@ ifneq ($(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS)),) SRCS += $(SRC_DIR)/tcu/tcu_tbuf.cpp endif +# Add MUFU extension sources +ifneq ($(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS)),) + SRCS += $(SRC_DIR)/mufu_unit.cpp + SRCS += $(SW_COMMON_DIR)/mufu.cpp +endif + # Add VM sources ifneq ($(filter -DVX_CFG_VM_ENABLE, $(XCONFIGS)),) SRCS += $(SRC_DIR)/mem/mmu.cpp diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 9369f2cbcb..c000ef2fa5 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -398,48 +398,47 @@ void CsrUnit::process(instr_trace_t* trace) { uint32_t num_threads = VX_CFG_NUM_THREADS; auto& rs1_data = trace->src_data[0]; - uint32_t thread_start = 0; - for (; thread_start < num_threads; ++thread_start) { - if (tmask.test(thread_start)) break; - } - trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; - switch (csr_type) { - case CsrType::CSRRW: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; - this->set_csr(csr_addr, src_v, wid, t); - rd_data[t].i = csr_value; + // The warp is processed one SFU-width packet at a time: a packet's active + // lanes read the CSR, then its first active lane supplies the write. + constexpr uint32_t kPacketLanes = VX_CFG_NUM_SFU_LANES; + for (uint32_t base = 0; base < num_threads; base += kPacketLanes) { + uint32_t first = num_threads; + for (uint32_t t = base; t < base + kPacketLanes; ++t) { + if (tmask.test(t)) { + first = t; + break; + } } - break; - case CsrType::CSRRS: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; - if (src_v != 0) { - this->set_csr(csr_addr, csr_value | src_v, wid, t); + if (first == num_threads) { + continue; + } + for (uint32_t t = first; t < base + kPacketLanes; ++t) { + if (tmask.test(t)) { + rd_data[t].i = this->get_csr(csr_addr, wid, t); } - rd_data[t].i = csr_value; } - break; - case CsrType::CSRRC: - for (uint32_t t = thread_start; t < num_threads; ++t) { - if (!tmask.test(t)) continue; - Word csr_value = this->get_csr(csr_addr, wid, t); - auto src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[t].i; + Word csr_value = rd_data[first].i; + Word src_v = csrArgs.is_imm ? csrArgs.imm : rs1_data[first].i; + switch (csr_type) { + case CsrType::CSRRW: + this->set_csr(csr_addr, src_v, wid, first); + break; + case CsrType::CSRRS: + if (src_v != 0) { + this->set_csr(csr_addr, csr_value | src_v, wid, first); + } + break; + case CsrType::CSRRC: if (src_v != 0) { - this->set_csr(csr_addr, csr_value & ~src_v, wid, t); + this->set_csr(csr_addr, csr_value & ~src_v, wid, first); } - rd_data[t].i = csr_value; + break; + default: + std::abort(); } - break; - default: - std::abort(); } DT(3, "csr-unit process: op=" << csr_type << ", " << *trace); } diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 9fb835db56..630ae1bbd9 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -219,7 +219,15 @@ static op_string_t op_string(const Instr &instr) { std::abort(); } } - case FpuType::F2F: return {fpuArgs.is_f64 ? "FCVT.D.S":"FCVT.S.D", ""}; + case FpuType::F2F: { + if (fpuArgs.is_f16) { + return {(fpuArgs.cvt == 1) ? "FCVT.H.D":"FCVT.H.S", ""}; + } + if (fpuArgs.cvt == 2) { + return {fpuArgs.is_f64 ? "FCVT.D.H":"FCVT.S.H", ""}; + } + return {fpuArgs.is_f64 ? "FCVT.D.S":"FCVT.S.D", ""}; + } case FpuType::FCMP: { switch (fpuArgs.frm) { case 0: return {fpuArgs.is_f64 ? "FLE.D":"FLE.S", ""}; @@ -239,8 +247,8 @@ static op_string_t op_string(const Instr &instr) { } } case FpuType::FCLASS: return {fpuArgs.is_f64 ? "FCLASS.D":"FCLASS.S", ""}; - case FpuType::FMVXW: return {fpuArgs.is_f64 ? "FMV.X.D":"FMV.X.S", ""}; - case FpuType::FMVWX: return {fpuArgs.is_f64 ? "FMV.D.X":"FMV.S.X", ""}; + case FpuType::FMVXW: return {fpuArgs.is_f16 ? "FMV.X.H" : fpuArgs.is_f64 ? "FMV.X.D":"FMV.X.S", ""}; + case FpuType::FMVWX: return {fpuArgs.is_f16 ? "FMV.H.X" : fpuArgs.is_f64 ? "FMV.D.X":"FMV.S.X", ""}; case FpuType::FMINMAX: { switch (fpuArgs.frm) { case 0: return {fpuArgs.is_f64 ? "FMIN.D":"FMIN.S", ""}; @@ -258,11 +266,15 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); - // packLD detection: Float dest with sub-word width (LB or LH). - if (dst_float && lsuArgs.width == 0) return {"PACKLB.F", ""}; - if (dst_float && lsuArgs.width == 1) return {"PACKLH.F", ""}; + if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { + return {"PACKLB.F", ""}; + } + if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) { + return {"PACKLH.F", ""}; + } if (dst_float) { switch (lsuArgs.width) { + case 1: return {"FLH", to_hex_str(lsuArgs.offset)}; case 2: return {"FLW", to_hex_str(lsuArgs.offset)}; case 3: return {"FLD", to_hex_str(lsuArgs.offset)}; default: @@ -288,6 +300,7 @@ static op_string_t op_string(const Instr &instr) { bool src_float = (instr.get_src_reg(1).type == RegType::Float); if (src_float) { switch (lsuArgs.width) { + case 1: return {"FSH", to_hex_str(lsuArgs.offset)}; case 2: return {"FSW", to_hex_str(lsuArgs.offset)}; case 3: return {"FSD", to_hex_str(lsuArgs.offset)}; default: @@ -404,6 +417,13 @@ static op_string_t op_string(const Instr &instr) { return TcuUnit::op_string(tcu_type, tpuArgs); } #endif // VX_CFG_EXT_TCU_ENABLE + #ifdef VX_CFG_EXT_MUFU_ENABLE + ,[&](MufuType mufu_type)-> op_string_t { + std::ostringstream os; + os << mufu_type; + return {os.str(), ""}; + } + #endif #ifdef VX_CFG_EXT_TEX_ENABLE ,[&](TexType /*tex_type*/)-> op_string_t { return {"TEX", ""}; @@ -703,7 +723,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { } break; case Opcode::FCI: { instr->set_fu_type(FUType::FPU); - instr->set_args(IntrFpuArgs{funct3, rs2, (funct7 & 0x1)}); + instr->set_args(IntrFpuArgs{funct3, rs2, (funct7 & 0x1), ((funct7 & 0x3) == 0x2)}); // Most FP ops raise exception flags; a dynamic rounding mode reads frm. uint8_t frm_rd = (funct3 == 0x7) ? Instr::FCSR_FRM : 0; switch (funct7) { @@ -754,13 +774,39 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_src_reg(1, rs2, RegType::Float); instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); break; - case 0x20: // FCVT.S.D - case 0x21: // FCVT.D.S + case 0x20: // FCVT.S.D, FCVT.S.H + case 0x21: // FCVT.D.S, FCVT.D.H + case 0x22: { // FCVT.H.S, FCVT.H.D + // fmt codes: 0 = S, 1 = D, 2 = H + uint32_t dst_fmt = funct7 & 0x3; + uint32_t src_fmt = rs2; + bool supported = (src_fmt <= 2) && (src_fmt != dst_fmt); +#ifndef VX_CFG_EXT_ZFHMIN_ENABLE + supported = supported && (dst_fmt != 2) && (src_fmt != 2); +#endif + if (!supported) { + std::abort(); + } instr->set_op_type(FpuType::F2F); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); instr->set_fcsr_use(frm_rd, Instr::FCSR_FFLAGS); + } break; +#ifdef VX_CFG_EXT_ZFHMIN_ENABLE + case 0x72: // FMV.X.H + if (funct3 != 0) { + std::abort(); + } + instr->set_op_type(FpuType::FMVXW); + instr->set_dest_reg(rd, RegType::Integer); + instr->set_src_reg(0, rs1, RegType::Float); break; + case 0x7a: // FMV.H.X + instr->set_op_type(FpuType::FMVWX); + instr->set_dest_reg(rd, RegType::Float); + instr->set_src_reg(0, rs1, RegType::Integer); + break; +#endif case 0x2c: // FSQRT.S case 0x2d: // FSQRT.D instr->set_op_type(FpuType::FSQRT); @@ -816,7 +862,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_op_type((op == Opcode::FMADD) ? FpuType::FMADD : (op == Opcode::FMSUB) ? FpuType::FMSUB : (op == Opcode::FNMADD) ? FpuType::FNMADD : FpuType::FNMSUB); - instr->set_args(IntrFpuArgs{funct3, funct2, (funct7 & 0x1)}); + instr->set_args(IntrFpuArgs{funct3, funct2, (funct7 & 0x1), 0}); instr->set_dest_reg(rd, RegType::Float); instr->set_src_reg(0, rs1, RegType::Float); instr->set_src_reg(1, rs2, RegType::Float); @@ -1144,6 +1190,17 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { // known-violation handoff class, checked against its decoded destination. gfx_doctrine::check(*instr); } break; +#ifdef VX_CFG_EXT_MUFU_ENABLE + case Opcode::EXT3: { + if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) { + std::abort(); + } + instr->set_fu_type(FUType::SFU); + instr->set_op_type(MufuType(rs2)); + instr->set_dest_reg(rd, RegType::Float); + instr->set_src_reg(0, rs1, RegType::Float); + } break; +#endif default: std::abort(); } diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 270f7b8a3f..6316092c57 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "debug.h" #include "core.h" #include "csr_unit.h" @@ -27,15 +28,16 @@ using namespace vortex; namespace { inline uint64_t nan_box(uint32_t value) { - return value | 0xffffffff00000000; -} -inline bool is_nan_boxed(uint64_t value) { - return (uint32_t(value >> 32) == 0xffffffff); + return nanbox::box32(value); } inline int64_t check_boxing(int64_t a) { - if (is_nan_boxed(a)) - return a; - return nan_box(0x7fc00000); // NaN + return nanbox::box32(nanbox::unbox32(a, 0x7fc00000)); +} +inline uint64_t nan_box16(uint16_t value) { + return nanbox::box16(value); +} +inline uint16_t check_boxing16(uint64_t a) { + return nanbox::unbox16(a, 0x7e00); } } @@ -271,7 +273,20 @@ void FpuUnit::execute(instr_trace_t* trace) { if (!tmask.test(t)) continue; uint32_t frm = emu.get_fpu_rm(fpuArgs.frm, wid, t); uint32_t fflags = 0; - if (fpuArgs.is_f64) { + if (fpuArgs.is_f16) { + if (fpuArgs.cvt == 1) { + rd_data[t].u64 = nan_box16(rv_dtoh(rs1_data[t].u64, frm, &fflags)); + } else { + rd_data[t].u64 = nan_box16(rv_ftoh_s(check_boxing(rs1_data[t].u64), frm, &fflags)); + } + } else if (fpuArgs.cvt == 2) { + auto h = check_boxing16(rs1_data[t].u64); + if (fpuArgs.is_f64) { + rd_data[t].u64 = rv_htod(h, frm, &fflags); + } else { + rd_data[t].u64 = nan_box(rv_htof_s(h, frm, &fflags)); + } + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rv_ftod(check_boxing(rs1_data[t].u64), frm, &fflags); } else { rd_data[t].u64 = nan_box(rv_dtof(rs1_data[t].u64, frm, &fflags)); @@ -295,7 +310,9 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; - if (fpuArgs.is_f64) { + if (fpuArgs.is_f16) { + rd_data[t].i = sext((uint64_t)uint16_t(rs1_data[t].u64), 16); + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].u64; } else { uint32_t result = (uint32_t)rs1_data[t].u64; @@ -308,7 +325,9 @@ void FpuUnit::execute(instr_trace_t* trace) { for (uint32_t t = thread_start; t < num_threads; ++t) { if (!tmask.test(t)) continue; uint32_t fflags = 0; - if (fpuArgs.is_f64) { + if (fpuArgs.is_f16) { + rd_data[t].u64 = nan_box16(uint16_t(rs1_data[t].i)); + } else if (fpuArgs.is_f64) { rd_data[t].u64 = rs1_data[t].i; } else { rd_data[t].u64 = nan_box((uint32_t)rs1_data[t].i); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index 78fd3fd466..c891fc3f51 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "debug.h" #include "core.h" #include "local_mem_switch.h" @@ -253,6 +254,7 @@ void LsuUnit::process_response_step(uint32_t b) { uint32_t byte_off = (data_bs == 0 || data_bs == 0x0F) ? 0 : (uint32_t)__builtin_ctz(data_bs); bool nan_box = dst_float && (data_bytes < 8); + bool half_float = VX_CFG_EXT_ZFHMIN_ENABLED && dst_float && (entry.lsu_args.width == 1); for (uint32_t lane = 0; lane < lsu_rsp.mask.size(); ++lane) { if (!lsu_rsp.mask.test(lane)) @@ -266,9 +268,12 @@ void LsuUnit::process_response_step(uint32_t b) { uint64_t formatted = 0; switch (entry.lsu_args.width) { case 0: // LB - case 1: // LH formatted = (uint64_t)(int64_t)sext((Word)read_data, data_width); break; + case 1: // LH (raw bits for a Zfhmin Float dest, NaN-boxed below) + formatted = half_float ? read_data + : (uint64_t)(int64_t)sext((Word)read_data, data_width); + break; case 2: // LW (sign-ext for Integer dest; raw bits for Float dest, NaN-boxed below) formatted = dst_float ? read_data : (uint64_t)(int64_t)sext((Word)read_data, data_width); @@ -284,8 +289,9 @@ void LsuUnit::process_response_step(uint32_t b) { } // Place at bytesel-specified position; OR-in NaN-box for Float dest. auto& dst = entry.trace->dst_data.at(lane_info.tid); + uint32_t box_bits = half_float ? 16 : 32; dst.u64 = (formatted << (8 * byte_off)) - | (nan_box ? 0xFFFFFFFF00000000ull : 0); + | (nan_box ? nanbox::box(0, box_bits) : 0); } } entry.count -= lsu_rsp.mask.count(); // track remaining diff --git a/sim/simx/mufu_unit.cpp b/sim/simx/mufu_unit.cpp new file mode 100644 index 0000000000..79a3f52ef3 --- /dev/null +++ b/sim/simx/mufu_unit.cpp @@ -0,0 +1,61 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mufu_unit.h" +#include +#include +#include "debug.h" + +using namespace vortex; + +uint32_t MufuUnit::issue_cycles(const instr_trace_t* trace) { + constexpr uint32_t kPacketLanes = VX_CFG_NUM_SFU_LANES; + constexpr uint32_t kBatches = VX_CFG_NUM_SFU_LANES / VX_CFG_NUM_MUFU_LANES; + static_assert(VX_CFG_NUM_SFU_LANES % VX_CFG_NUM_MUFU_LANES == 0, + "MUFU lanes must divide the SFU lanes"); + uint32_t cycles = 0; + for (uint32_t base = 0; base < VX_CFG_NUM_THREADS; base += kPacketLanes) { + for (uint32_t t = base; t < base + kPacketLanes; ++t) { + if (trace->tmask.test(t)) { + cycles += kBatches; + break; + } + } + } + return cycles; +} + +bool MufuUnit::process(instr_trace_t* trace) { + if (head_ != trace) { + head_ = trace; + remaining_ = issue_cycles(trace); + } + if (remaining_ > 1) { + --remaining_; + return false; + } + head_ = nullptr; + remaining_ = 0; + + auto fn = uint32_t(std::get(trace->op_type)); + auto& rs1_data = trace->src_data[0]; + trace->dst_data.assign(VX_CFG_NUM_THREADS, reg_data_t{}); + for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { + if (trace->tmask.test(t)) { + uint32_t x = nanbox::unbox32(rs1_data[t].u64, mufu::QNAN_F32); + trace->dst_data[t].u64 = nanbox::box32(mufu::eval(fn, x)); + } + } + DT(3, "mufu-unit process: " << *trace); + return true; +} diff --git a/sim/simx/mufu_unit.h b/sim/simx/mufu_unit.h new file mode 100644 index 0000000000..6e4e780c1d --- /dev/null +++ b/sim/simx/mufu_unit.h @@ -0,0 +1,41 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include "types.h" +#include "instr_trace.h" + +namespace vortex { + +// MUFU sub-unit of the SFU. The SFU's lanes are serialized onto the MUFU +// datapath lanes, one batch per cycle, and a warp is accepted on its last batch. +class MufuUnit { +public: + // Cycles a result spends in the datapath after the warp is accepted. + static constexpr uint32_t kLatency = VX_MUFU_LATENCY; + + // Returns false while the warp's batches are still being issued; on the + // accepting cycle computes the results and returns true. + bool process(instr_trace_t* trace); + +private: + static uint32_t issue_cycles(const instr_trace_t* trace); + + const instr_trace_t* head_ = nullptr; + uint32_t remaining_ = 0; +}; + +} diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index de61756629..2b680b1132 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -390,6 +390,21 @@ void SfuUnit::on_tick() { // The fragment work distributor (above + scheduler) launches fragment // warps directly from the autonomously-pulled covered-quad waves. +#ifdef VX_CFG_EXT_MUFU_ENABLE + if (std::get_if(&trace->op_type)) { + if (output.full()) { + continue; + } + if (!mufu_unit_.process(trace)) { + continue; + } + output.send(trace, this->latency_of(trace) + MufuUnit::kLatency); + trace->resume_warp = trace->fetch_stall; + input.pop(); + continue; + } +#endif + if (output.full()) continue; // stall — no side effects this tick diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 1eb74a31f1..9485f6bc19 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -35,6 +35,9 @@ #ifdef VX_CFG_EXT_RTU_ENABLE #include "rtu/rtu_unit.h" #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +#include "mufu_unit.h" +#endif namespace vortex { @@ -159,6 +162,9 @@ class SfuUnit : public FuncUnit { #ifdef VX_CFG_EXT_RTU_ENABLE std::unique_ptr rtu_unit_; #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE + MufuUnit mufu_unit_; +#endif }; } diff --git a/sim/simx/types.h b/sim/simx/types.h index 1658771c6e..31f7fbddbf 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -562,6 +562,7 @@ struct IntrFpuArgs { uint32_t frm : 3; uint32_t cvt : 2; uint32_t is_f64 : 1; + uint32_t is_f16 : 1; }; inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { @@ -592,6 +593,36 @@ inline std::ostream &operator<<(std::ostream &os, const FpuType& type) { /////////////////////////////////////////////////////////////////////////////// +#ifdef VX_CFG_EXT_MUFU_ENABLE +// Values match the instruction's fn field and vortex::mufu::fn_t. +enum class MufuType { + RCP = 0, + RSQRT = 1, + EX2 = 2, + LG2 = 3, + SINPI = 4, + COSPI = 5, + TANH = 6, +}; + +inline std::ostream &operator<<(std::ostream &os, const MufuType& type) { + switch (type) { + case MufuType::RCP: os << "MUFU.RCP"; break; + case MufuType::RSQRT: os << "MUFU.RSQRT"; break; + case MufuType::EX2: os << "MUFU.EX2"; break; + case MufuType::LG2: os << "MUFU.LG2"; break; + case MufuType::SINPI: os << "MUFU.SINPI"; break; + case MufuType::COSPI: os << "MUFU.COSPI"; break; + case MufuType::TANH: os << "MUFU.TANH"; break; + default: + assert(false); + } + return os; +} +#endif + +/////////////////////////////////////////////////////////////////////////////// + enum class WctlType { TMC, WSPAWN, @@ -821,6 +852,9 @@ using OpType = std::variant< #ifdef VX_CFG_EXT_TCU_ENABLE , TcuType #endif +#ifdef VX_CFG_EXT_MUFU_ENABLE +, MufuType +#endif #ifdef VX_CFG_EXT_TEX_ENABLE , TexType #endif diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 4e51883d19..df068a9785 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -130,6 +130,11 @@ ifneq (,$(filter -DVX_CFG_EXT_RTU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/rtu endif +# Add MUFU extension sources +ifneq (,$(filter -DVX_CFG_EXT_MUFU_ENABLE, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/mufu +endif + RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Command Processor: declare the package only. Interfaces live in diff --git a/sw/common/mufu.cpp b/sw/common/mufu.cpp new file mode 100644 index 0000000000..2896183dda --- /dev/null +++ b/sw/common/mufu.cpp @@ -0,0 +1,246 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mufu.h" +#include "mufu_tables.h" + +namespace vortex::mufu { + +namespace { + +constexpr uint32_t FRAC = 30; +constexpr uint32_t XL_BITS = 18; + +constexpr uint32_t POS_INF = 0x7f800000; +constexpr uint32_t NEG_INF = 0xff800000; +constexpr uint32_t ONE = 0x3f800000; + +// Output of the range-reduction stage. +struct reduced_t { + bool special; // result is `value`, bypassing interpolation + uint32_t value; + uint32_t addr; // coefficient ROM address + uint32_t xl; // XL_BITS-wide position inside the interval + int32_t exp; // result scale: value = P * 2^-FRAC * 2^exp + bool neg; // result sign + bool zsign; // sign of an exactly-zero result + bool is_lg2; // add `k` as the integer part before normalizing + int32_t k; +}; + +reduced_t reduce(uint32_t fn, uint32_t x) { + reduced_t r{}; + uint32_t s = x >> 31; + uint32_t e = (x >> 23) & 0xff; + uint32_t m = x & 0x7fffff; + bool is_zero = (e == 0); + bool is_inf = (e == 0xff) && (m == 0); + bool is_nan = (e == 0xff) && (m != 0); + uint32_t sig = m | (1u << 23); + + auto special = [&](uint32_t v) { r.special = true; r.value = v; return r; }; + + if (is_nan) { + return special(QNAN_F32); + } + + switch (fn) { + case FN_RCP: + if (is_inf) { + return special(s << 31); + } + if (is_zero) { + return special(s ? NEG_INF : POS_INF); + } + r.addr = TBL_RCP_BASE + (m >> 16); + r.xl = (m & 0xffff) << 2; + r.exp = 127 - int32_t(e); + r.neg = s; + return r; + + case FN_RSQRT: { + if (is_zero) { + return special(s ? NEG_INF : POS_INF); + } + if (s) { + return special(QNAN_F32); + } + if (is_inf) { + return special(0); + } + int32_t k = int32_t(e) - 127; + r.addr = TBL_RSQRT_BASE + ((uint32_t(k) & 1) << 6) + (m >> 17); + r.xl = (m & 0x1ffff) << 1; + r.exp = -(k >> 1); + return r; + } + + case FN_EX2: { + if (is_inf) { + return special(s ? 0 : POS_INF); + } + if (is_zero) { + return special(ONE); + } + if (e >= 134) { + return special(s ? 0 : POS_INF); + } + int32_t sh = int32_t(e) - 120; + int64_t mag = (sh >= 0) ? (int64_t(sig) << sh) : ((sh > -24) ? int64_t(sig >> -sh) : 0); + int64_t X = s ? -mag : mag; + int64_t f = X & ((int64_t(1) << FRAC) - 1); + r.addr = TBL_EX2_BASE + uint32_t(f >> 24); + r.xl = uint32_t(f >> 6) & ((1u << XL_BITS) - 1); + r.exp = int32_t(X >> FRAC); + return r; + } + + case FN_LG2: + if (is_zero) { + return special(NEG_INF); + } + if (s) { + return special(QNAN_F32); + } + if (is_inf) { + return special(POS_INF); + } + r.addr = TBL_LG2_BASE + (m >> 17); + r.xl = (m & 0x1ffff) << 1; + r.is_lg2 = true; + r.k = int32_t(e) - 127; + return r; + + case FN_SINPI: + case FN_COSPI: { + bool is_sin = (fn == FN_SINPI); + if (is_inf) { + return special(QNAN_F32); + } + if (is_zero) { + return special(is_sin ? (s << 31) : ONE); + } + // u = 2|x| in quarter turns, 30 fraction bits, modulo 4 + int32_t sh = int32_t(e) - 119; + uint32_t u; + if (sh >= 32) { + u = 0; + } else if (sh >= 0) { + u = uint32_t(uint64_t(sig) << sh); + } else { + u = (sh > -24) ? (sig >> -sh) : 0; + } + if (!is_sin) { + u += (1u << FRAC); + } + uint32_t q = u >> FRAC; + uint32_t t = u & ((1u << FRAC) - 1); + r.neg = ((q >> 1) & 1) ^ (is_sin ? s : 0); + r.zsign = is_sin && s; + // Odd quadrants need cos(pi/2 t) = sin(pi/2 (1 - t)); ~t stands in for + // 1 - t, and t == 0 (exactly 1) is returned directly to stay exact. + if (q & 1) { + if (t == 0) { + return special(r.neg ? (ONE | 0x80000000) : ONE); + } + t = ~t & ((1u << FRAC) - 1); + } + r.addr = TBL_SIN_BASE + (t >> 24); + r.xl = (t >> 6) & ((1u << XL_BITS) - 1); + return r; + } + + case FN_TANH: + if (is_inf) { + return special(s ? (ONE | 0x80000000) : ONE); + } + if (is_zero) { + return special(s << 31); + } + // |x| >= 8 rounds to +-1; below 2^-5, tanh x = x within 2^-11. + if (e >= 130) { + return special(s ? (ONE | 0x80000000) : ONE); + } + if (e < 122) { + return special(x); + } + // Entries hold 2^-(j+4) * tanh(2^j (1+t)) for binade j = e - 127. + r.addr = TBL_TANH_BASE + ((e - 122) << 3) + (m >> 20); + r.xl = (m >> 2) & ((1u << XL_BITS) - 1); + r.exp = int32_t(e) - 127 + 4; + r.neg = s; + return r; + + default: + return special(0); + } +} + +int64_t interpolate(uint32_t addr, uint32_t xl) { + const coeff_t& c = kTable[addr]; + int64_t a0 = int64_t(c.a0) << S0; + int64_t a1 = int64_t(c.a1) * (int64_t(1) << S1); + int64_t a2 = int64_t(c.a2) * (int64_t(1) << S2); + int64_t sq = (int64_t(xl) * xl) >> XL_BITS; + int64_t t1 = (a1 * xl) >> XL_BITS; + int64_t t2 = (a2 * sq) >> XL_BITS; + int64_t p = a0 + t1 + t2; + return (p < 0) ? 0 : p; +} + +uint32_t normalize(uint64_t mag, int32_t exp, bool neg, bool zsign) { + if (mag == 0) { + return uint32_t(zsign) << 31; + } + int lead = 63 - __builtin_clzll(mag); + uint64_t norm = (mag << (39 - lead)) & ((uint64_t(1) << 40) - 1); + uint32_t sig = uint32_t(norm >> 16); + uint32_t rnd = uint32_t(norm >> 15) & 1; + int32_t bexp = lead - int32_t(FRAC) + exp + 127; + sig += rnd; + if (sig >> 24) { + sig >>= 1; + bexp += 1; + } + uint32_t sgn = uint32_t(neg) << 31; + if (bexp >= 255) { + return sgn | POS_INF; + } + if (bexp <= 0) { + return sgn; + } + return sgn | (uint32_t(bexp) << 23) | (sig & 0x7fffff); +} + +} // namespace + +uint32_t eval(uint32_t fn, uint32_t x) { + reduced_t r = reduce(fn, x); + if (r.special) { + return r.value; + } + int64_t p = interpolate(r.addr, r.xl); + if (r.is_lg2) { + int64_t l = (int64_t(r.k) << FRAC) + p; + bool neg = (l < 0); + return normalize(uint64_t(neg ? -l : l), 0, neg, false); + } + return normalize(uint64_t(p), r.exp, r.neg, r.zsign); +} + +const char* fn_name(uint32_t fn) { + static const char* names[FN_COUNT] = {"rcp", "rsqrt", "ex2", "lg2", "sinpi", "cospi", "tanh"}; + return (fn < FN_COUNT) ? names[fn] : "?"; +} + +} // namespace vortex::mufu diff --git a/sw/common/mufu.h b/sw/common/mufu.h new file mode 100644 index 0000000000..f633d07cf3 --- /dev/null +++ b/sw/common/mufu.h @@ -0,0 +1,38 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +// Bit-exact reference for the MUFU special functions. +namespace vortex::mufu { + +enum fn_t : uint32_t { + FN_RCP = 0, + FN_RSQRT = 1, + FN_EX2 = 2, + FN_LG2 = 3, + FN_SINPI = 4, + FN_COSPI = 5, + FN_TANH = 6, + FN_COUNT = 7 +}; + +constexpr uint32_t QNAN_F32 = 0x7fc00000; + +uint32_t eval(uint32_t fn, uint32_t x); + +const char* fn_name(uint32_t fn); + +} // namespace vortex::mufu diff --git a/sw/common/mufu_tables.h b/sw/common/mufu_tables.h new file mode 100644 index 0000000000..31a1276b32 --- /dev/null +++ b/sw/common/mufu_tables.h @@ -0,0 +1,538 @@ +// Generated by hw/scripts/gen_mufu_tables.py. Do not edit. +#pragma once + +#include + +namespace vortex::mufu { + +// Stored coefficients; the datapath uses a0 << S0, a1 << S1, a2 << S2. +struct coeff_t { uint32_t a0; int32_t a1; int32_t a2; }; + +constexpr uint32_t S0 = 2; +constexpr uint32_t S1 = 2; +constexpr uint32_t S2 = 1; + +constexpr uint32_t TBL_RCP_BASE = 0; +constexpr uint32_t TBL_RSQRT_BASE = 128; +constexpr uint32_t TBL_EX2_BASE = 256; +constexpr uint32_t TBL_LG2_BASE = 320; +constexpr uint32_t TBL_SIN_BASE = 384; +constexpr uint32_t TBL_TANH_BASE = 448; +constexpr uint32_t TBL_SIZE = 512; + +constexpr coeff_t kTable[TBL_SIZE] = { + {0x10000000u, -2097098, 32415}, + {0x0fe03f7du, -2064695, 31643}, + {0x0fc0fc0cu, -2033054, 30921}, + {0x0fa232ccu, -2002134, 30221}, + {0x0f83e0f5u, -1971915, 29542}, + {0x0f6603d6u, -1942374, 28883}, + {0x0f4898d3u, -1913493, 28244}, + {0x0f2b9d61u, -1885250, 27623}, + {0x0f0f0f0cu, -1857629, 27020}, + {0x0ef2eb6fu, -1830610, 26435}, + {0x0ed73038u, -1804176, 25867}, + {0x0ebbdb28u, -1778311, 25314}, + {0x0ea0ea0cu, -1752998, 24778}, + {0x0e865ac6u, -1728222, 24256}, + {0x0e6c2b42u, -1703967, 23749}, + {0x0e525980u, -1680219, 23256}, + {0x0e38e38cu, -1656965, 22776}, + {0x0e1fc77eu, -1634189, 22310}, + {0x0e070380u, -1611880, 21856}, + {0x0dee95c2u, -1590025, 21415}, + {0x0dd67c88u, -1568611, 20985}, + {0x0dbeb61cu, -1547627, 20567}, + {0x0da740d9u, -1527062, 20159}, + {0x0d901b1eu, -1506903, 19763}, + {0x0d79435cu, -1487141, 19377}, + {0x0d62b80cu, -1467765, 19000}, + {0x0d4c77aeu, -1448766, 18634}, + {0x0d3680d2u, -1430133, 18277}, + {0x0d20d20cu, -1411857, 17928}, + {0x0d0b69fbu, -1393929, 17589}, + {0x0cf64749u, -1376341, 17258}, + {0x0ce168a6u, -1359083, 16936}, + {0x0cccccccu, -1342148, 16621}, + {0x0cb8727bu, -1325528, 16314}, + {0x0ca4587du, -1309214, 16015}, + {0x0c907da4u, -1293200, 15723}, + {0x0c7ce0c6u, -1277477, 15438}, + {0x0c6980c6u, -1262040, 15160}, + {0x0c565c87u, -1246881, 14888}, + {0x0c4372f7u, -1231993, 14623}, + {0x0c30c30bu, -1217370, 14364}, + {0x0c1e4bbcu, -1203006, 14112}, + {0x0c0c0c0bu, -1188895, 13865}, + {0x0bfa02fdu, -1175031, 13624}, + {0x0be82fa0u, -1161408, 13388}, + {0x0bd69103u, -1148020, 13158}, + {0x0bc52640u, -1134863, 12933}, + {0x0bb3ee71u, -1121930, 12713}, + {0x0ba2e8b9u, -1109217, 12498}, + {0x0b92143fu, -1096720, 12288}, + {0x0b81702du, -1084432, 12083}, + {0x0b70fbb5u, -1072349, 11882}, + {0x0b60b60au, -1060467, 11686}, + {0x0b509e68u, -1048782, 11494}, + {0x0b40b40au, -1037289, 11306}, + {0x0b30f634u, -1025984, 11122}, + {0x0b21642cu, -1014862, 10942}, + {0x0b11fd3au, -1003920, 10766}, + {0x0b02c0b0u, -993155, 10594}, + {0x0af3addcu, -982561, 10425}, + {0x0ae4c416u, -972137, 10260}, + {0x0ad602b5u, -961877, 10098}, + {0x0ac76918u, -951779, 9940}, + {0x0ab8f69eu, -941839, 9785}, + {0x0aaaaaaau, -932053, 9634}, + {0x0a9c84a4u, -922420, 9485}, + {0x0a8e83f5u, -912935, 9340}, + {0x0a80a80au, -903596, 9197}, + {0x0a72f053u, -894399, 9057}, + {0x0a655c43u, -885342, 8920}, + {0x0a57eb50u, -876422, 8786}, + {0x0a4a9cf1u, -867636, 8655}, + {0x0a3d70a3u, -858981, 8526}, + {0x0a3065e4u, -850456, 8400}, + {0x0a237c32u, -842056, 8276}, + {0x0a16b313u, -833781, 8154}, + {0x0a0a0a0au, -825627, 8035}, + {0x09fd809fu, -817591, 7919}, + {0x09f1165eu, -809673, 7804}, + {0x09e4cad2u, -801869, 7692}, + {0x09d89d89u, -794177, 7582}, + {0x09cc8e16u, -786596, 7474}, + {0x09c09c0au, -779123, 7368}, + {0x09b4c6fau, -771755, 7264}, + {0x09a90e7eu, -764492, 7162}, + {0x099d722du, -757330, 7061}, + {0x0991f1a5u, -750269, 6963}, + {0x09868c80u, -743306, 6867}, + {0x097b425fu, -736440, 6772}, + {0x097012e0u, -729668, 6679}, + {0x0964fda6u, -722989, 6588}, + {0x095a0256u, -716402, 6498}, + {0x094f2095u, -709904, 6410}, + {0x09445809u, -703494, 6324}, + {0x0939a85cu, -697170, 6239}, + {0x092f1138u, -690932, 6155}, + {0x09249249u, -684777, 6073}, + {0x091a2b3cu, -678703, 5993}, + {0x090fdbc1u, -672711, 5914}, + {0x0905a386u, -666797, 5836}, + {0x08fb823eu, -660960, 5760}, + {0x08f1779eu, -655201, 5685}, + {0x08e78357u, -649516, 5611}, + {0x08dda520u, -643904, 5539}, + {0x08d3dcb0u, -638365, 5468}, + {0x08ca29c0u, -632898, 5398}, + {0x08c08c09u, -627500, 5329}, + {0x08b70344u, -622171, 5262}, + {0x08ad8f30u, -616910, 5195}, + {0x08a42f87u, -611715, 5130}, + {0x089ae408u, -606585, 5065}, + {0x0891ac74u, -601520, 5002}, + {0x08888888u, -596517, 4940}, + {0x087f7808u, -591577, 4879}, + {0x08767ab6u, -586698, 4819}, + {0x086d9054u, -581880, 4760}, + {0x0864b8a8u, -577120, 4702}, + {0x085bf376u, -572418, 4644}, + {0x08534085u, -567774, 4588}, + {0x084a9f9cu, -563186, 4533}, + {0x08421084u, -558654, 4478}, + {0x08399305u, -554175, 4425}, + {0x083126eau, -549751, 4372}, + {0x0828cbfcu, -545379, 4320}, + {0x08208208u, -541059, 4269}, + {0x081848dbu, -536791, 4218}, + {0x08102040u, -532572, 4169}, + {0x08080808u, -528404, 4120}, + {0x10000000u, -2097017, 48275}, + {0x0fe05ebbu, -2048776, 46389}, + {0x0fc1763cu, -2002395, 44664}, + {0x0fa33f8cu, -1957737, 43028}, + {0x0f85b41du, -1914715, 41475}, + {0x0f68cdb4u, -1873246, 39999}, + {0x0f4c8666u, -1833252, 38596}, + {0x0f30d896u, -1794662, 37260}, + {0x0f15bee9u, -1757407, 35989}, + {0x0efb3448u, -1721422, 34777}, + {0x0ee133dau, -1686650, 33622}, + {0x0ec7b8ffu, -1653032, 32519}, + {0x0eaebf50u, -1620517, 31467}, + {0x0e964295u, -1589053, 30462}, + {0x0e7e3ecdu, -1558595, 29501}, + {0x0e66b01fu, -1529098, 28582}, + {0x0e4f92dfu, -1500519, 27702}, + {0x0e38e38au, -1472819, 26860}, + {0x0e229ec5u, -1445962, 26054}, + {0x0e0cc155u, -1419910, 25281}, + {0x0df74828u, -1394632, 24539}, + {0x0de23046u, -1370095, 23828}, + {0x0dcd76dau, -1346269, 23145}, + {0x0db9192bu, -1323126, 22490}, + {0x0da5149bu, -1300638, 21860}, + {0x0d9166a8u, -1278780, 21254}, + {0x0d7e0ceau, -1257528, 20672}, + {0x0d6b050cu, -1236857, 20112}, + {0x0d584cd4u, -1216747, 19573}, + {0x0d45e21eu, -1197176, 19053}, + {0x0d33c2d8u, -1178124, 18553}, + {0x0d21ed03u, -1159572, 18071}, + {0x0d105eb6u, -1141502, 17607}, + {0x0cff1616u, -1123896, 17159}, + {0x0cee115du, -1106739, 16727}, + {0x0cdd4ed1u, -1090013, 16310}, + {0x0ccccccbu, -1073705, 15907}, + {0x0cbc89b0u, -1057799, 15518}, + {0x0cac83f4u, -1042282, 15142}, + {0x0c9cba1au, -1027140, 14779}, + {0x0c8d2aafu, -1012362, 14428}, + {0x0c7dd44fu, -997935, 14089}, + {0x0c6eb5a1u, -983847, 13760}, + {0x0c5fcd57u, -970088, 13442}, + {0x0c511a2du, -956646, 13135}, + {0x0c429aebu, -943512, 12837}, + {0x0c344e63u, -930676, 12549}, + {0x0c26336eu, -918128, 12269}, + {0x0c1848f2u, -905859, 11998}, + {0x0c0a8ddbu, -893862, 11736}, + {0x0bfd011fu, -882127, 11481}, + {0x0befa1bau, -870646, 11234}, + {0x0be26eb2u, -859413, 10995}, + {0x0bd56714u, -848418, 10762}, + {0x0bc889f5u, -837657, 10537}, + {0x0bbbd66eu, -827120, 10318}, + {0x0baf4ba2u, -816803, 10105}, + {0x0ba2e8bau, -806699, 9898}, + {0x0b96ace2u, -796801, 9697}, + {0x0b8a974fu, -787104, 9502}, + {0x0b7ea73cu, -777602, 9313}, + {0x0b72dbe8u, -768290, 9128}, + {0x0b673498u, -759162, 8949}, + {0x0b5bb097u, -750214, 8774}, + {0x0b504f2du, -1482786, 34088}, + {0x0b39f191u, -1448703, 32802}, + {0x0b241696u, -1415907, 31582}, + {0x0b0eb956u, -1384329, 30426}, + {0x0af9d52eu, -1353908, 29327}, + {0x0ae565b7u, -1324585, 28284}, + {0x0ad166c1u, -1296305, 27291}, + {0x0abdd454u, -1269018, 26347}, + {0x0aaaaaa6u, -1242674, 25448}, + {0x0a97e620u, -1217230, 24591}, + {0x0a858351u, -1192642, 23774}, + {0x0a737ef6u, -1168870, 22995}, + {0x0a61d5f1u, -1145878, 22250}, + {0x0a508548u, -1123630, 21540}, + {0x0a3f8a25u, -1102093, 20860}, + {0x0a2ee1cfu, -1081235, 20210}, + {0x0a1e89aeu, -1061027, 19589}, + {0x0a0e7f48u, -1041441, 18993}, + {0x09fec039u, -1022449, 18423}, + {0x09ef4a3eu, -1004028, 17876}, + {0x09e01b26u, -986154, 17352}, + {0x09d130dau, -968803, 16849}, + {0x09c2895bu, -951956, 16366}, + {0x09b422b9u, -935591, 15903}, + {0x09a5fb1cu, -919690, 15457}, + {0x099810beu, -904234, 15029}, + {0x098a61eau, -889206, 14617}, + {0x097cecfdu, -874590, 14221}, + {0x096fb061u, -860370, 13840}, + {0x0962aa93u, -846531, 13473}, + {0x0955da1du, -833060, 13119}, + {0x09493d95u, -819941, 12778}, + {0x093cd3a2u, -807164, 12450}, + {0x09309af3u, -794715, 12133}, + {0x09249248u, -782582, 11828}, + {0x0918b868u, -770756, 11533}, + {0x090d0c28u, -759224, 11248}, + {0x09018c65u, -747977, 10973}, + {0x08f63808u, -737004, 10707}, + {0x08eb0e02u, -726298, 10450}, + {0x08e00d4fu, -715848, 10202}, + {0x08d534f2u, -705647, 9962}, + {0x08ca83f6u, -695685, 9730}, + {0x08bff96fu, -685956, 9505}, + {0x08b5947au, -676451, 9288}, + {0x08ab5439u, -667164, 9077}, + {0x08a137d5u, -658087, 8873}, + {0x08973e80u, -649214, 8676}, + {0x088d6772u, -640539, 8484}, + {0x0883b1e7u, -632056, 8298}, + {0x087a1d22u, -623758, 8118}, + {0x0870a86eu, -615640, 7944}, + {0x08675318u, -607696, 7774}, + {0x085e1c75u, -599922, 7610}, + {0x085503deu, -592313, 7451}, + {0x084c08b1u, -584862, 7296}, + {0x08432a51u, -577567, 7145}, + {0x083a6824u, -570422, 6999}, + {0x0831c199u, -563424, 6857}, + {0x0829361cu, -556567, 6719}, + {0x0820c524u, -549848, 6585}, + {0x08186e27u, -543263, 6455}, + {0x081030a2u, -536809, 6328}, + {0x08080c14u, -530481, 6204}, + {0x10000000u, 2907245, 31646}, + {0x102c9a46u, 2938896, 32003}, + {0x1059b0dbu, 2970898, 32351}, + {0x10874520u, 3003249, 32704}, + {0x10b55875u, 3035952, 33060}, + {0x10e3ec3bu, 3069012, 33420}, + {0x111301d8u, 3102431, 33784}, + {0x11429ab7u, 3136214, 34151}, + {0x1172b845u, 3170365, 34523}, + {0x11a35bf4u, 3204888, 34899}, + {0x11d4873au, 3239787, 35279}, + {0x12063b91u, 3275066, 35663}, + {0x12387a77u, 3310729, 36052}, + {0x126b456fu, 3346781, 36444}, + {0x129e9dfeu, 3383225, 36841}, + {0x12d285b0u, 3420066, 37242}, + {0x1306fe13u, 3457308, 37648}, + {0x133c08bbu, 3494956, 38058}, + {0x1371a740u, 3533013, 38472}, + {0x13a7db3eu, 3571485, 38891}, + {0x13dea656u, 3610376, 39315}, + {0x14160a2cu, 3649690, 39743}, + {0x144e086au, 3689433, 40176}, + {0x1486a2c0u, 3729608, 40613}, + {0x14bfdadfu, 3770221, 41055}, + {0x14f9b281u, 3811276, 41502}, + {0x15342b61u, 3852778, 41954}, + {0x156f4741u, 3894732, 42411}, + {0x15ab07e8u, 3937143, 42873}, + {0x15e76f20u, 3980016, 43340}, + {0x16247ebbu, 4023355, 43812}, + {0x1662388du, 4067167, 44289}, + {0x16a09e71u, 4111455, 44771}, + {0x16dfb247u, 4156226, 45259}, + {0x171f75f4u, 4201484, 45752}, + {0x175feb61u, 4247236, 46250}, + {0x17a1147fu, 4293485, 46753}, + {0x17e2f342u, 4340238, 47263}, + {0x182589a5u, 4387500, 47777}, + {0x1868d9a7u, 4435277, 48297}, + {0x18ace54eu, 4483574, 48823}, + {0x18f1aea5u, 4532397, 49355}, + {0x193737bdu, 4581751, 49892}, + {0x197d82acu, 4631643, 50436}, + {0x19c4918fu, 4682078, 50985}, + {0x1a0c6688u, 4733063, 51540}, + {0x1a5503beu, 4784603, 52101}, + {0x1a9e6b62u, 4836704, 52669}, + {0x1ae89fa6u, 4889372, 53242}, + {0x1b33a2c5u, 4942614, 53822}, + {0x1b7f7700u, 4996435, 54408}, + {0x1bcc1e9du, 5050843, 55001}, + {0x1c199bebu, 5105843, 55599}, + {0x1c67f13cu, 5161442, 56205}, + {0x1cb720eau, 5217646, 56817}, + {0x1d072d58u, 5274463, 57436}, + {0x1d5818ebu, 5331898, 58061}, + {0x1da9e612u, 5389958, 58693}, + {0x1dfc9742u, 5448651, 59332}, + {0x1e502ef6u, 5507983, 59979}, + {0x1ea4afb1u, 5567961, 60632}, + {0x1efa1bfdu, 5628592, 61292}, + {0x1f50766au, 5689883, 61959}, + {0x1fa7c190u, 5751842, 62634}, + {0x00000000u, 6050894, -93196}, + {0x005b9e69u, 5957748, -90271}, + {0x00b5d6aau, 5867487, -87576}, + {0x010eb397u, 5779920, -85001}, + {0x01663f7cu, 5694928, -82537}, + {0x01bc8430u, 5612399, -80179}, + {0x02118b1du, 5532228, -77920}, + {0x02655d48u, 5454315, -75756}, + {0x02b80352u, 5378566, -73680}, + {0x03098584u, 5304892, -71689}, + {0x0359ebd0u, 5233209, -69777}, + {0x03a93dd3u, 5163437, -67941}, + {0x03f782e0u, 5095502, -66176}, + {0x0444c200u, 5029330, -64480}, + {0x049101f3u, 4964856, -62847}, + {0x04dc493cu, 4902013, -61276}, + {0x05269e1bu, 4840742, -59763}, + {0x05700696u, 4780983, -58305}, + {0x05b8887bu, 4722681, -56901}, + {0x06002960u, 4665784, -55546}, + {0x0646eeaau, 4610242, -54239}, + {0x068cdd8au, 4556007, -52978}, + {0x06d1fb04u, 4503033, -51760}, + {0x07164bf2u, 4451276, -50583}, + {0x0759d4feu, 4400696, -49447}, + {0x079c9aaeu, 4351252, -48348}, + {0x07dea160u, 4302907, -47285}, + {0x081fed4bu, 4255625, -46257}, + {0x08608286u, 4209370, -45263}, + {0x08a06502u, 4164110, -44300}, + {0x08df9894u, 4119813, -43367}, + {0x091e20efu, 4076448, -42463}, + {0x095c01a8u, 4033987, -41588}, + {0x09993e3au, 3992401, -40739}, + {0x09d5da02u, 3951664, -39916}, + {0x0a11d844u, 3911749, -39118}, + {0x0a4d3c2au, 3872633, -38343}, + {0x0a8808c8u, 3834292, -37591}, + {0x0ac24118u, 3796702, -36862}, + {0x0afbe7feu, 3759842, -36153}, + {0x0b35004bu, 3723691, -35464}, + {0x0b6d8cb9u, 3688228, -34795}, + {0x0ba58fefu, 3653435, -34145}, + {0x0bdd0c80u, 3619292, -33512}, + {0x0c1404eeu, 3585781, -32897}, + {0x0c4a7ba9u, 3552885, -32299}, + {0x0c80730eu, 3520587, -31717}, + {0x0cb5ed6cu, 3488871, -31151}, + {0x0ceaed02u, 3457721, -30600}, + {0x0d1f73fdu, 3427123, -30063}, + {0x0d53847eu, 3397061, -29540}, + {0x0d872096u, 3367522, -29031}, + {0x0dba4a4au, 3338493, -28534}, + {0x0ded0392u, 3309959, -28051}, + {0x0e1f4e54u, 3281910, -27579}, + {0x0e512c71u, 3254331, -27120}, + {0x0e829fb9u, 3227213, -26671}, + {0x0eb3a9f2u, 3200542, -26234}, + {0x0ee44cd8u, 3174309, -25808}, + {0x0f148a19u, 3148502, -25391}, + {0x0f44635cu, 3123111, -24985}, + {0x0f73da3bu, 3098127, -24588}, + {0x0fa2f048u, 3073539, -24201}, + {0x0fd1a70bu, 3049339, -23823}, + {0x00000000u, 6588682, -1841}, + {0x00648550u, 6586785, -5952}, + {0x00c8fb28u, 6580833, -9916}, + {0x012d5201u, 6570917, -13874}, + {0x01917a64u, 6557043, -17823}, + {0x01f564ddu, 6539219, -21762}, + {0x02590206u, 6517456, -25688}, + {0x02bc4280u, 6491767, -29598}, + {0x031f16ffu, 6462168, -33490}, + {0x03817045u, 6428676, -37362}, + {0x03e33f27u, 6391312, -41212}, + {0x04447491u, 6350097, -45037}, + {0x04a50184u, 6305058, -48835}, + {0x0504d71eu, 6256221, -52603}, + {0x0563e696u, 6203616, -56340}, + {0x05c22145u, 6147273, -60042}, + {0x061f78a2u, 6087228, -63709}, + {0x067bde4au, 6023516, -67337}, + {0x06d743fbu, 5956176, -70925}, + {0x07319ba0u, 5885247, -74470}, + {0x078ad747u, 5810774, -77970}, + {0x07e2e930u, 5732801, -81423}, + {0x0839c3c6u, 5651374, -84827}, + {0x088f59a4u, 5566543, -88180}, + {0x08e39d96u, 5478360, -91479}, + {0x093682a0u, 5386876, -94724}, + {0x0987fbf8u, 5292147, -97912}, + {0x09d7fd0eu, 5194231, -101040}, + {0x0a26798cu, 5093185, -104108}, + {0x0a736558u, 4989072, -107113}, + {0x0abeb494u, 4881954, -110054}, + {0x0b085ba4u, 4771895, -112928}, + {0x0b504f2eu, 4658961, -115734}, + {0x0b968416u, 4543221, -118471}, + {0x0bdaef8cu, 4424744, -121136}, + {0x0c1d8700u, 4303603, -123728}, + {0x0c5e4031u, 4179868, -126246}, + {0x0c9d1120u, 4053616, -128688}, + {0x0cd9f020u, 3924922, -131052}, + {0x0d14d3ccu, 3793864, -133337}, + {0x0d4db310u, 3660521, -135542}, + {0x0d848528u, 3524973, -137665}, + {0x0db9419eu, 3387301, -139705}, + {0x0debe052u, 3247589, -141661}, + {0x0e1c5975u, 3105921, -143532}, + {0x0e4aa58du, 2962382, -145317}, + {0x0e76bd76u, 2817059, -147013}, + {0x0ea09a66u, 2670038, -148622}, + {0x0ec835e5u, 2521409, -150140}, + {0x0eed89d9u, 2371262, -151569}, + {0x0f109080u, 2219686, -152906}, + {0x0f314474u, 2066773, -154151}, + {0x0f4fa0a9u, 1912615, -155303}, + {0x0f6ba072u, 1757305, -156361}, + {0x0f853f7cu, 1600937, -157325}, + {0x0f9c79d4u, 1443604, -158195}, + {0x0fb14be7u, 1285401, -158969}, + {0x0fc3b27cu, 1126425, -159648}, + {0x0fd3aabeu, 966770, -160230}, + {0x0fe13238u, 806532, -160716}, + {0x0fec46d1u, 645809, -161105}, + {0x0ff4e6d6u, 484696, -161397}, + {0x0ffb10f1u, 323292, -161591}, + {0x0ffec430u, 161693, -161689}, + {0x00ffeab1u, 2095111, -543}, + {0x011fe1a8u, 2094568, -607}, + {0x013fd660u, 2093961, -670}, + {0x015fc899u, 2093291, -734}, + {0x017fb814u, 2092557, -797}, + {0x019fa492u, 2091759, -861}, + {0x01bf8dd2u, 2090898, -924}, + {0x01df7396u, 2089974, -987}, + {0x00ffaad0u, 2089005, -2163}, + {0x011f86c0u, 2086842, -2414}, + {0x013f59c0u, 2084427, -2664}, + {0x015f22d5u, 2081763, -2912}, + {0x017ee105u, 2078851, -3160}, + {0x019e9359u, 2075691, -3405}, + {0x01be38dbu, 2072286, -3649}, + {0x01ddd096u, 2068636, -3891}, + {0x00feacc8u, 2064812, -8502}, + {0x011e1dcfu, 2056309, -9447}, + {0x013d6bc7u, 2046860, -10374}, + {0x015c9307u, 2036484, -11282}, + {0x017b8ff8u, 2025201, -12169}, + {0x019a5f1bu, 2013030, -13033}, + {0x01b8fd04u, 1999995, -13874}, + {0x01d7665cu, 1986119, -14689}, + {0x00facbe9u, 1971642, -31722}, + {0x0118a38fu, 1939902, -34656}, + {0x0135f980u, 1905227, -37353}, + {0x0152c2bdu, 1867854, -39803}, + {0x016ef537u, 1828031, -41997}, + {0x018a87dcu, 1786014, -43931}, + {0x01a5729au, 1742062, -45605}, + {0x01bfae68u, 1696437, -47018}, + {0x00ec9a8fu, 1649657, -97285}, + {0x01050869u, 1552225, -99966}, + {0x011bf481u, 1452129, -100861}, + {0x013157eau, 1351155, -100166}, + {0x0145324fu, 1250893, -98128}, + {0x01578914u, 1152689, -94986}, + {0x0168666au, 1057644, -90980}, + {0x0177d854u, 966623, -86335}, + {0x00c2f848u, 878759, -157188}, + {0x00cf2eceu, 721543, -135503}, + {0x00d9298au, 586134, -114288}, + {0x00e13cb5u, 472007, -94752}, + {0x00e7b821u, 377446, -77498}, + {0x00ece3c5u, 300142, -62714}, + {0x00f0fe38u, 237609, -50328}, + {0x00f43c80u, 187441, -40124}, + {0x007b65ceu, 145562, -56829}, + {0x007d304fu, 89495, -35261}, + {0x007e4998u, 54733, -21685}, + {0x007ef565u, 33365, -13263}, + {0x007f5e0au, 20299, -8086}, + {0x007f9dacu, 12335, -4920}, + {0x007fc453u, 7490, -2989}, + {0x007fdbcbu, 4546, -1815}, + {0x003ff50cu, 2635, -1750}, + {0x003ffbf8u, 970, -644}, + {0x003ffe84u, 357, -237}, + {0x003fff75u, 131, -87}, + {0x003fffcdu, 48, -32}, + {0x003fffedu, 18, -12}, + {0x003ffff9u, 7, -4}, + {0x003ffffeu, 2, -2}, +}; + +} // namespace vortex::mufu diff --git a/sw/common/nan_box.h b/sw/common/nan_box.h new file mode 100644 index 0000000000..4d68737a9b --- /dev/null +++ b/sw/common/nan_box.h @@ -0,0 +1,56 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +// RISC-V NaN-boxing: a narrow floating-point value held in a wider FP register +// has every register bit above it set to one; a value that is not boxed reads +// as the narrow format's canonical NaN. +namespace vortex::nanbox { + +constexpr uint64_t low_mask(uint32_t bits) { + return (bits >= 64) ? ~uint64_t(0) : ((uint64_t(1) << bits) - 1); +} + +constexpr uint64_t box(uint64_t value, uint32_t bits, uint32_t reg_bits = 64) { + return ((value & low_mask(bits)) | ~low_mask(bits)) & low_mask(reg_bits); +} + +constexpr bool is_boxed(uint64_t value, uint32_t bits, uint32_t reg_bits = 64) { + uint64_t upper = low_mask(reg_bits) & ~low_mask(bits); + return (value & upper) == upper; +} + +constexpr uint64_t unbox(uint64_t value, uint32_t bits, uint64_t nan, uint32_t reg_bits = 64) { + return is_boxed(value, bits, reg_bits) ? (value & low_mask(bits)) : nan; +} + +constexpr uint64_t box32(uint32_t value) { + return box(value, 32); +} + +constexpr uint32_t unbox32(uint64_t value, uint32_t nan) { + return uint32_t(unbox(value, 32, nan)); +} + +constexpr uint64_t box16(uint16_t value) { + return box(value, 16); +} + +constexpr uint16_t unbox16(uint64_t value, uint16_t nan) { + return uint16_t(unbox(value, 16, nan)); +} + +} diff --git a/sw/common/rvfloats.cpp b/sw/common/rvfloats.cpp index 11762bccd8..607ae472d7 100644 --- a/sw/common/rvfloats.cpp +++ b/sw/common/rvfloats.cpp @@ -548,6 +548,20 @@ uint16_t rv_ftoh_s(uint32_t a, uint32_t frm, uint32_t* fflags) { return from_float16_t(r); } +uint64_t rv_htod(uint16_t a, uint32_t frm, uint32_t* fflags) { + rv_init(frm); + auto r = f16_to_f64(to_float16_t(a)); + if (fflags) { *fflags = softfloat_exceptionFlags; } + return from_float64_t(r); +} + +uint16_t rv_dtoh(uint64_t a, uint32_t frm, uint32_t* fflags) { + rv_init(frm); + auto r = f64_to_f16(to_float64_t(a)); + if (fflags) { *fflags = softfloat_exceptionFlags; } + return from_float16_t(r); +} + uint32_t rv_btof_s(uint16_t a, uint32_t frm, uint32_t* fflags) { rv_init(frm); auto r = bf16_to_f32(to_bfloat16_t(a)); diff --git a/sw/common/rvfloats.h b/sw/common/rvfloats.h index 57d03079a5..6b7192b805 100644 --- a/sw/common/rvfloats.h +++ b/sw/common/rvfloats.h @@ -94,6 +94,8 @@ uint64_t rv_ftod(uint32_t a, uint32_t frm, uint32_t* fflags); // fp16 <--> fp32 conversions uint32_t rv_htof_s(uint16_t a, uint32_t frm, uint32_t* fflags); uint16_t rv_ftoh_s(uint32_t a, uint32_t frm, uint32_t* fflags); +uint64_t rv_htod(uint16_t a, uint32_t frm, uint32_t* fflags); +uint16_t rv_dtoh(uint64_t a, uint32_t frm, uint32_t* fflags); // bf16 <--> fp32 conversions uint32_t rv_btof_s(uint16_t a, uint32_t frm, uint32_t* fflags); diff --git a/sw/kernel/include/vx_mufu.h b/sw/kernel/include/vx_mufu.h new file mode 100644 index 0000000000..7a236d64c1 --- /dev/null +++ b/sw/kernel/include/vx_mufu.h @@ -0,0 +1,114 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// MUFU special-function instructions (custom-2, requires VX_CFG_EXT_MUFU_ENABLE). +// Results are approximate: subnormals +// flush to zero, frm is ignored and fflags are never raised. +// +// Encoding: .insn r CUSTOM2, fmt=funct3, 0=funct7, fd, fs1, x +// The function select rides the rs2 field, so it is written as a literal xN. + +#ifndef __VX_MUFU_H__ +#define __VX_MUFU_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define __VX_MUFU_S(name, fn) \ +__attribute__((always_inline)) \ +inline float name(float x) { \ + float r; \ + __asm__ (".insn r %1, 0, 0, %0, %2, x" #fn \ + : "=f"(r) : "i"(RISCV_CUSTOM2), "f"(x)); \ + return r; \ +} + +__VX_MUFU_S(vx_rcpf, 0) // 1/x +__VX_MUFU_S(vx_rsqrtf, 1) // 1/sqrt(x) +__VX_MUFU_S(vx_ex2f, 2) // 2^x +__VX_MUFU_S(vx_lg2f, 3) // log2(x) +__VX_MUFU_S(vx_sinpif, 4) // sin(pi*x) +__VX_MUFU_S(vx_cospif, 5) // cos(pi*x) +__VX_MUFU_S(vx_tanhf, 6) // tanh(x) + +#undef __VX_MUFU_S + +// Fast-math compositions, as CUDA's __expf/__logf/__sinf/__cosf build on MUFU. + +__attribute__((always_inline)) +inline float vx_expf_fast(float x) { + return vx_ex2f(x * 1.44269504088896341f); +} + +__attribute__((always_inline)) +inline float vx_logf_fast(float x) { + return vx_lg2f(x) * 0.693147180559945309f; +} + +__attribute__((always_inline)) +inline float vx_sinf_fast(float x) { + return vx_sinpif(x * 0.318309886183790672f); +} + +__attribute__((always_inline)) +inline float vx_cosf_fast(float x) { + return vx_cospif(x * 0.318309886183790672f); +} + +__attribute__((always_inline)) +inline float vx_sqrtf_fast(float x) { + // x * rsqrt(x) is 0 * inf at both ends, and MUFU flushes subnormal x to zero. + // The fixups use integer selects: a float select lowers to a branch, which + // is not divergence-safe. + float r = x * vx_rsqrtf(x); + uint32_t xb, rb; + __builtin_memcpy(&xb, &x, sizeof(xb)); + __builtin_memcpy(&rb, &r, sizeof(rb)); + rb = ((xb & 0x7f800000) == 0) ? (xb & 0x80000000) : rb; + rb = (xb == 0x7f800000) ? xb : rb; + __builtin_memcpy(&r, &rb, sizeof(r)); + return r; +} + +__attribute__((always_inline)) +inline float vx_divf_fast(float a, float b) { + return a * vx_rcpf(b); +} + +// Activations. + +__attribute__((always_inline)) +inline float vx_sigmoidf_fast(float x) { + return 0.5f + 0.5f * vx_tanhf(0.5f * x); +} + +__attribute__((always_inline)) +inline float vx_siluf_fast(float x) { + return x * vx_sigmoidf_fast(x); +} + +// GELU, tanh form: 0.5x(1 + tanh(sqrt(2/pi)(x + 0.044715x^3))). +__attribute__((always_inline)) +inline float vx_geluf_fast(float x) { + float u = 0.797884560802865356f * x * (1.0f + 0.044715f * x * x); + return 0.5f * x * (1.0f + vx_tanhf(u)); +} + +#ifdef __cplusplus +} +#endif + +#endif // __VX_MUFU_H__ diff --git a/sw/runtime/include/vortex2.h b/sw/runtime/include/vortex2.h index 0f79760b9c..be8916cce3 100644 --- a/sw/runtime/include/vortex2.h +++ b/sw/runtime/include/vortex2.h @@ -109,6 +109,8 @@ typedef struct vx_kernel* vx_kernel_h; #define VX_ISA_EXT_TCU (1ull << (32 + 9)) #define VX_ISA_EXT_DXA (1ull << (32 + 10)) #define VX_ISA_EXT_RTU (1ull << (32 + 11)) +#define VX_ISA_EXT_MUFU (1ull << (32 + 12)) +#define VX_ISA_EXT_ZFHMIN (1ull << (32 + 13)) // ============================================================================ // Device memory access flags (vx_buffer_create / vx_buffer_access) diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e50ebfff69..e5be8fed12 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ arith printf diverge diverge_large sort fence ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax \ + softmax mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ diff --git a/tests/regression/common.mk b/tests/regression/common.mk index ed7a588772..25ffaf4462 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -45,11 +45,17 @@ else ZACAS_EXT := endif +ifneq (,$(filter -DVX_CFG_EXT_ZFHMIN_ENABLE, $(XCONFIGS))) + ZFHMIN_EXT := _zfhmin +else + ZFHMIN_EXT := +endif + ifeq ($(XLEN),64) - VX_CFLAGS += -march=rv64imafd$(C_EXT)$(ZACAS_EXT) -mabi=lp64d + VX_CFLAGS += -march=rv64imafd$(C_EXT)$(ZACAS_EXT)$(ZFHMIN_EXT) -mabi=lp64d STARTUP_ADDR ?= 0x180000000 else - VX_CFLAGS += -march=rv32imaf$(C_EXT)$(ZACAS_EXT) -mabi=ilp32f + VX_CFLAGS += -march=rv32imaf$(C_EXT)$(ZACAS_EXT)$(ZFHMIN_EXT) -mabi=ilp32f STARTUP_ADDR ?= 0x80000000 endif diff --git a/tests/regression/mufu/Makefile b/tests/regression/mufu/Makefile new file mode 100644 index 0000000000..cdd955d62e --- /dev/null +++ b/tests/regression/mufu/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_MUFU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_MUFU_ENABLE) + +PROJECT := mufu + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp $(SW_COMMON_DIR)/mufu.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +VX_HDRS := $(VORTEX_HOME)/sw/kernel/include/vx_mufu.h $(SRC_DIR)/common.h + +OPTS ?= -n4096 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/mufu/common.h b/tests/regression/mufu/common.h new file mode 100644 index 0000000000..18930ab03c --- /dev/null +++ b/tests/regression/mufu/common.h @@ -0,0 +1,36 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Ops 0..6 are the raw MUFU functions (same numbering as the instruction's fn +// field); the rest are the fast-math compositions from vx_mufu.h. +enum { + OP_RCP = 0, + OP_RSQRT, + OP_EX2, + OP_LG2, + OP_SINPI, + OP_COSPI, + OP_TANH, + OP_EXPF, + OP_LOGF, + OP_SINF, + OP_COSF, + OP_SQRTF, + OP_SIGMOID, + OP_SILU, + OP_GELU, + OP_COUNT +}; + +typedef struct { + uint32_t op; + uint32_t num_points; + uint32_t repeat; + uint32_t padding; + uint64_t src_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/mufu/kernel.cpp b/tests/regression/mufu/kernel.cpp new file mode 100644 index 0000000000..54e5760b02 --- /dev/null +++ b/tests/regression/mufu/kernel.cpp @@ -0,0 +1,46 @@ +#include +#include +#include "common.h" + +template +static void apply(const float* src, float* dst, uint32_t n, uint32_t repeat, F f) { + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) { + return; + } + float x = src[i]; + float y = f(x); + // Extra rounds recompute f(x) to lengthen the run; the empty asm statements + // hide x and keep each result live so the rounds cannot be folded. + for (uint32_t r = 1; r < repeat; ++r) { + __asm__ volatile ("" : "+f"(x)); + y = f(x); + __asm__ volatile ("" :: "f"(y)); + } + dst[i] = y; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + uint32_t n = arg->num_points; + uint32_t k = arg->repeat; + switch (arg->op) { + case OP_RCP: apply(src, dst, n, k, vx_rcpf); break; + case OP_RSQRT: apply(src, dst, n, k, vx_rsqrtf); break; + case OP_EX2: apply(src, dst, n, k, vx_ex2f); break; + case OP_LG2: apply(src, dst, n, k, vx_lg2f); break; + case OP_SINPI: apply(src, dst, n, k, vx_sinpif); break; + case OP_COSPI: apply(src, dst, n, k, vx_cospif); break; + case OP_TANH: apply(src, dst, n, k, vx_tanhf); break; + case OP_EXPF: apply(src, dst, n, k, vx_expf_fast); break; + case OP_LOGF: apply(src, dst, n, k, vx_logf_fast); break; + case OP_SINF: apply(src, dst, n, k, vx_sinf_fast); break; + case OP_COSF: apply(src, dst, n, k, vx_cosf_fast); break; + case OP_SQRTF: apply(src, dst, n, k, vx_sqrtf_fast); break; + case OP_SIGMOID: apply(src, dst, n, k, vx_sigmoidf_fast); break; + case OP_SILU: apply(src, dst, n, k, vx_siluf_fast); break; + case OP_GELU: apply(src, dst, n, k, vx_geluf_fast); break; + default: break; + } +} diff --git a/tests/regression/mufu/main.cpp b/tests/regression/mufu/main.cpp new file mode 100644 index 0000000000..337f12ebeb --- /dev/null +++ b/tests/regression/mufu/main.cpp @@ -0,0 +1,342 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +static const uint32_t kSqrtSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x7f800000, +}; + +static const char* kOpNames[OP_COUNT] = { + "rcp", "rsqrt", "ex2", "lg2", "sinpi", "cospi", "tanh", + "expf", "logf", "sinf", "cosf", "sqrtf", "sigmoid", "silu", "gelu" +}; + +static const uint32_t kSpecials[] = { + 0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x80800000, + 0x3f800000, 0xbf800000, 0x3f000000, 0x40000000, 0x40800000, 0x40490fdb, + 0x7f7fffff, 0xff7fffff, 0x7f800000, 0xff800000, 0x7fc00000, 0x7fa00000, + 0x42fe0000, 0x43000000, 0xc2fc0000, 0xc3000000, 0x4b000001, 0x3f7fffff, + 0x3f800001, 0x39800000, 0x41000000, 0x40ffffff, +}; + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_points = 4096; +uint32_t repeat = 1; +int selected_op = -1; + +vx_device_h device = nullptr; +vx_buffer_h src_buffer = nullptr; +vx_buffer_h dst_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; + +static void show_usage() { + std::cout << "Vortex MUFU Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n points] [-r repeat] [-t op] [-h: help]" << std::endl; + std::cout << " ops:"; + for (auto name : kOpNames) { + std::cout << " " << name; + } + std::cout << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:r:t:k:h")) != -1) { + switch (c) { + case 'n': + num_points = atoi(optarg); + break; + case 'r': + repeat = std::max(1, atoi(optarg)); + break; + case 't': + for (int i = 0; i < OP_COUNT; ++i) { + if (strcmp(optarg, kOpNames[i]) == 0) { + selected_op = i; + } + } + if (selected_op < 0) { + show_usage(); + exit(-1); + } + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (src_buffer) { + vx_buffer_release(src_buffer); + } + if (dst_buffer) { + vx_buffer_release(dst_buffer); + } + if (kernel) { + vx_kernel_release(kernel); + } + if (module_) { + vx_module_release(module_); + } + if (queue) { + vx_queue_release(queue); + } + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +static float as_float(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static uint32_t as_bits(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} + +static float rand_range(std::mt19937& rng, float lo, float hi) { + return std::uniform_real_distribution(lo, hi)(rng); +} + +// Log-uniform magnitude in [2^lo, 2^hi], random sign when `sign`. +static float rand_log(std::mt19937& rng, float lo, float hi, bool sign) { + float m = std::exp2(rand_range(rng, lo, hi)); + return (sign && (rng() & 1)) ? -m : m; +} + +static float gen_input(int op, std::mt19937& rng) { + // A quarter of raw-op inputs are arbitrary bit patterns. + if (op <= OP_TANH && (rng() & 3) == 0) { + return as_float(uint32_t(rng())); + } + switch (op) { + case OP_RCP: return rand_log(rng, -125, 125, true); + case OP_RSQRT: return rand_log(rng, -125, 125, false); + case OP_EX2: return rand_range(rng, -130.0f, 130.0f); + case OP_LG2: return (rng() & 1) ? rand_range(rng, 0.5f, 2.0f) : rand_log(rng, -125, 125, false); + case OP_SINPI: + case OP_COSPI: return (rng() & 1) ? rand_range(rng, -2.0f, 2.0f) : rand_range(rng, -4096.0f, 4096.0f); + case OP_TANH: return (rng() & 1) ? rand_range(rng, -10.0f, 10.0f) : rand_log(rng, -20, 3, true); + case OP_EXPF: return rand_range(rng, -80.0f, 80.0f); + case OP_LOGF: return rand_log(rng, -120, 120, false); + case OP_SINF: + case OP_COSF: return rand_range(rng, -100.0f, 100.0f); + case OP_SQRTF: return rand_log(rng, -120, 120, false); + case OP_SIGMOID: + case OP_SILU: + case OP_GELU: return rand_range(rng, -20.0f, 20.0f); + default: return 0.0f; + } +} + +// Raw-op accuracy against libm, with the bounds the MUFU spec guarantees. +static bool raw_accurate(int op, float x, float y) { + double r; + switch (op) { + case OP_RCP: r = 1.0 / x; break; + case OP_RSQRT: r = 1.0 / std::sqrt((double)x); break; + case OP_EX2: r = std::exp2((double)x); break; + case OP_LG2: r = std::log2((double)x); break; + case OP_SINPI: r = std::sin(M_PI * std::fmod((double)x, 2.0)); break; + case OP_COSPI: r = std::cos(M_PI * std::fmod((double)x, 2.0)); break; + case OP_TANH: r = std::tanh((double)x); break; + default: return true; + } + // Out of scope for the bound: specials, and results the FTZ/overflow rules decide. + if (!std::isnormal(x) || std::isnan(r) || std::isinf(r) || std::fabs(r) >= 3.4028234663852886e38 || std::fabs(r) < 1.1754943508222875e-38) { + return true; + } + double err = std::fabs((double)y - r); + int ex; + std::frexp(r, &ex); + double ulp = std::ldexp(1.0, ex - 24); + switch (op) { + case OP_LG2: return (x >= 0.5f && x <= 2.0f) ? err <= 0x1p-22 : err <= 2 * ulp; + case OP_SINPI: + case OP_COSPI: return err <= 0x1p-21; + case OP_TANH: return err <= 0x1p-11 * std::fabs(r); + default: return err <= 2 * ulp; + } +} + +// Composite reference and tolerance; the composition rounds its prescale, so +// the bound grows with |x| where the prescale is a multiply. +static bool composite_accurate(int op, float x, float y) { + // Subnormal inputs flush to a zero of the same sign. + if (op == OP_SQRTF && !std::isnormal(x)) { + float ref = std::isinf(x) ? x : x * 0.0f; + return as_bits(y) == as_bits(ref); + } + double xd = x, r, tol; + switch (op) { + case OP_EXPF: r = std::exp(xd); tol = (2e-7 + 1e-7 * std::fabs(xd)) * std::fabs(r); break; + case OP_LOGF: r = std::log(xd); tol = 1e-6 + 4e-7 * std::fabs(r); break; + case OP_SINF: r = std::sin(xd); tol = 1e-6 + 1e-7 * std::fabs(xd); break; + case OP_COSF: r = std::cos(xd); tol = 1e-6 + 1e-7 * std::fabs(xd); break; + case OP_SQRTF: r = std::sqrt(xd); tol = 4e-7 * r; break; + // tanh is accurate to 2^-11 relative; 1/2 + tanh/2 turns that into 2^-12 + // absolute (small sigmoid values are not relatively accurate). + case OP_SIGMOID: r = 1.0 / (1.0 + std::exp(-xd)); tol = 0x1p-12; break; + case OP_SILU: r = xd / (1.0 + std::exp(-xd)); tol = 0x1p-12 * std::max(1.0, std::fabs(xd)); break; + case OP_GELU: r = 0.5 * xd * (1.0 + std::tanh(0.7978845608028654 * (xd + 0.044715 * xd * xd * xd))); + tol = 0x1p-12 * std::max(1.0, std::fabs(xd)); break; + default: return true; + } + return std::fabs((double)y - r) <= tol; +} + +static int run_op(int op, std::mt19937& rng, uint64_t src_addr, uint64_t dst_addr) { + std::vector h_src(num_points); + std::vector h_dst(num_points); + const uint32_t* specials = nullptr; + size_t ns = 0; + if (op <= OP_TANH) { + specials = kSpecials; + ns = sizeof(kSpecials) / sizeof(kSpecials[0]); + } else if (op == OP_SQRTF) { + specials = kSqrtSpecials; + ns = sizeof(kSqrtSpecials) / sizeof(kSqrtSpecials[0]); + } + ns = std::min(num_points, ns); + for (uint32_t i = 0; i < num_points; ++i) { + h_src[i] = (i < ns) ? as_float(specials[i]) : gen_input(op, rng); + } + + kernel_arg_t arg = {}; + arg.op = op; + arg.num_points = num_points; + arg.repeat = repeat; + arg.src_addr = src_addr; + arg.dst_addr = dst_addr; + + uint32_t buf_size = num_points * sizeof(float); + RT_CHECK(vx_enqueue_write(queue, src_buffer, 0, h_src.data(), buf_size, 0, nullptr, nullptr)); + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &num_points, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + RT_CHECK(vx_enqueue_read(queue, h_dst.data(), dst_buffer, 0, buf_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + for (uint32_t i = 0; i < num_points; ++i) { + float x = h_src[i], y = h_dst[i]; + bool ok; + if (op <= OP_TANH) { + uint32_t ref = vortex::mufu::eval(op, as_bits(x)); + ok = (as_bits(y) == ref) && raw_accurate(op, x, y); + if (!ok && errors < 20) { + printf("*** error: %s[%u] x=%a (0x%08x): actual=0x%08x expected=0x%08x\n", + kOpNames[op], i, x, as_bits(x), as_bits(y), ref); + } + } else { + ok = composite_accurate(op, x, y); + if (!ok && errors < 20) { + printf("*** error: %s[%u] x=%a (0x%08x): actual=%a\n", kOpNames[op], i, x, as_bits(x), y); + } + } + errors += !ok; + } + std::cout << kOpNames[op] << ": " << num_points << " points, " + << errors << " errors" << std::endl; + return errors; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t isa_flags; + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + if ((isa_flags & VX_ISA_EXT_MUFU) == 0) { + std::cout << "MUFU extension not supported!" << std::endl; + cleanup(); + return -1; + } + + uint32_t buf_size = num_points * sizeof(float); + uint64_t src_addr, dst_addr; + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_READ, &src_buffer)); + RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_WRITE, &dst_buffer)); + RT_CHECK(vx_buffer_address(dst_buffer, &dst_addr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + std::mt19937 rng(0x6d756675); + int errors = 0; + for (int op = 0; op < OP_COUNT; ++op) { + if (selected_op >= 0 && op != selected_op) { + continue; + } + errors += run_op(op, rng, src_addr, dst_addr); + } + + std::cout << "cleanup" << std::endl; + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/zfhmin/Makefile b/tests/regression/zfhmin/Makefile new file mode 100644 index 0000000000..bba9c53cf3 --- /dev/null +++ b/tests/regression/zfhmin/Makefile @@ -0,0 +1,20 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_ZFHMIN_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_ZFHMIN_ENABLE) + +PROJECT := zfhmin + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include + +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/zfhmin/common.h b/tests/regression/zfhmin/common.h new file mode 100644 index 0000000000..2dff7131fd --- /dev/null +++ b/tests/regression/zfhmin/common.h @@ -0,0 +1,48 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define ZF_NUM_RM 6 + +enum { + ZF_MODE_NARROW_S = 0, + ZF_MODE_WIDEN_S = 1, + ZF_MODE_NARROW_D = 2, + ZF_MODE_WIDEN_D = 3, + ZF_MODE_FLAGS = 4, + ZF_MODE_BOX = 5, +}; + +enum { + ZF_OP_H_S = 0, + ZF_OP_S_H = 1, + ZF_OP_H_D = 2, + ZF_OP_D_H = 3, +}; + +#define ZF_BOX_OUTS 5 + +typedef struct { + uint32_t op; + uint32_t rm; + uint64_t bits; +} zf_flag_case_t; + +typedef struct { + uint64_t result; + uint32_t fflags; + uint32_t pad; +} zf_flag_out_t; + +typedef struct { + uint32_t mode; + uint32_t count; + uint32_t frm; + uint32_t pad; + uint64_t src_addr; + uint64_t dst_addr; + uint64_t aux_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/zfhmin/kernel.cpp b/tests/regression/zfhmin/kernel.cpp new file mode 100644 index 0000000000..6cc5b9a8ae --- /dev/null +++ b/tests/regression/zfhmin/kernel.cpp @@ -0,0 +1,182 @@ +#include +#include +#include "common.h" + +#if __riscv_xlen == 64 +typedef uint64_t raw_t; +#define FMV_X_RAW "fmv.x.d" +#else +typedef uint32_t raw_t; +#define FMV_X_RAW "fmv.x.w" +#endif + +#define CVT(insn, out, in, rm) __asm__ volatile(insn " %0, %1, " rm : "=f"(out) : "f"(in)) + +static inline __attribute__((always_inline)) void store_h(uint16_t* p, float h) { + __asm__ volatile("fsh %0, 0(%1)" :: "f"(h), "r"(p) : "memory"); +} + +static inline __attribute__((always_inline)) float load_h(const uint16_t* p) { + float h; + __asm__ volatile("flh %0, 0(%1)" : "=f"(h) : "r"(p) : "memory"); + return h; +} + +static inline __attribute__((always_inline)) long mv_x_h(float h) { + long r; + __asm__ volatile("fmv.x.h %0, %1" : "=r"(r) : "f"(h)); + return r; +} + +static inline __attribute__((always_inline)) float mv_h_x(long x) { + float h; + __asm__ volatile("fmv.h.x %0, %1" : "=f"(h) : "r"(x)); + return h; +} + +static inline __attribute__((always_inline)) float mv_w_x(long x) { + float f; + __asm__ volatile("fmv.w.x %0, %1" : "=f"(f) : "r"(x)); + return f; +} + +static inline __attribute__((always_inline)) raw_t mv_x_raw(float f) { + raw_t r; + __asm__ volatile(FMV_X_RAW " %0, %1" : "=r"(r) : "f"(f)); + return r; +} + +static inline __attribute__((always_inline)) uint32_t mv_x_w(float f) { + long r; + __asm__ volatile("fmv.x.w %0, %1" : "=r"(r) : "f"(f)); + return (uint32_t)r; +} + +static void narrow_s(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_NUM_RM; + auto mvx = reinterpret_cast(arg->aux_addr); + float x = src[i]; + float h; + CVT("fcvt.h.s", h, x, "rne"); store_h(dst + 0, h); mvx[i] = mv_x_h(h); + CVT("fcvt.h.s", h, x, "rtz"); store_h(dst + 1, h); + CVT("fcvt.h.s", h, x, "rdn"); store_h(dst + 2, h); + CVT("fcvt.h.s", h, x, "rup"); store_h(dst + 3, h); + CVT("fcvt.h.s", h, x, "rmm"); store_h(dst + 4, h); + CVT("fcvt.h.s", h, x, "dyn"); store_h(dst + 5, h); +} + +static void widen_s(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + float h = load_h(src + i); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(h)); + dst[i] = mv_x_w(y); +} + +#if __riscv_xlen == 64 +static void narrow_d(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_NUM_RM; + double x = src[i]; + float h; + CVT("fcvt.h.d", h, x, "rne"); store_h(dst + 0, h); + CVT("fcvt.h.d", h, x, "rtz"); store_h(dst + 1, h); + CVT("fcvt.h.d", h, x, "rdn"); store_h(dst + 2, h); + CVT("fcvt.h.d", h, x, "rup"); store_h(dst + 3, h); + CVT("fcvt.h.d", h, x, "rmm"); store_h(dst + 4, h); + CVT("fcvt.h.d", h, x, "dyn"); store_h(dst + 5, h); +} + +static void widen_d(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + float h = load_h(src + i); + double y; + __asm__ volatile("fcvt.d.h %0, %1" : "=f"(y) : "f"(h)); + uint64_t r; + __asm__ volatile("fmv.x.d %0, %1" : "=r"(r) : "f"(y)); + dst[i] = r; +} +#endif + +static void flag_case(const kernel_arg_t* arg, uint32_t i) { + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + uint32_t op = src[i].op; + uint64_t bits = src[i].bits; + uint64_t res = 0; + long fflags; + __asm__ volatile("fsrm %0" :: "r"(src[i].rm)); + __asm__ volatile("csrw fflags, zero"); + if (op == ZF_OP_H_S) { + float x = mv_w_x((long)(uint32_t)bits); + float h; + CVT("fcvt.h.s", h, x, "dyn"); + res = (uint16_t)mv_x_h(h); + } else if (op == ZF_OP_S_H) { + float h = mv_h_x((long)bits); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(h)); + res = mv_x_w(y); + } +#if __riscv_xlen == 64 + else if (op == ZF_OP_H_D) { + double x; + __asm__ volatile("fmv.d.x %0, %1" : "=f"(x) : "r"(bits)); + float h; + CVT("fcvt.h.d", h, x, "dyn"); + res = (uint16_t)mv_x_h(h); + } else if (op == ZF_OP_D_H) { + float h = mv_h_x((long)bits); + double y; + __asm__ volatile("fcvt.d.h %0, %1" : "=f"(y) : "f"(h)); + __asm__ volatile("fmv.x.d %0, %1" : "=r"(res) : "f"(y)); + } +#endif + __asm__ volatile("csrr %0, fflags" : "=r"(fflags)); + dst[i].result = res; + dst[i].fflags = (uint32_t)fflags; +} + +static void box(const kernel_arg_t* arg, uint32_t i) { + auto hsrc = reinterpret_cast(arg->src_addr); + auto gsrc = reinterpret_cast(arg->aux_addr); + auto dst = reinterpret_cast(arg->dst_addr) + i * ZF_BOX_OUTS; + long g = (long)gsrc[i]; + dst[0] = (uint64_t)mv_x_raw(load_h(hsrc + i)); + dst[1] = (uint64_t)mv_x_raw(mv_h_x(g)); + dst[2] = (uint64_t)(int64_t)mv_x_h(load_h(hsrc + i)); + float y; + __asm__ volatile("fcvt.s.h %0, %1" : "=f"(y) : "f"(mv_w_x(g))); + dst[3] = mv_x_w(y); + dst[4] = (uint64_t)(int64_t)mv_x_h(mv_w_x(g)); +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (arg->mode == ZF_MODE_FLAGS) { + int nt = vx_num_threads(); + for (int l = 0; l < nt; ++l) { + if (vx_thread_id() == l && i < arg->count) { + flag_case(arg, i); + } + } + return; + } + if (i >= arg->count) { + return; + } + __asm__ volatile("fsrm %0" ::"r"(arg->frm)); + switch (arg->mode) { + case ZF_MODE_NARROW_S: narrow_s(arg, i); break; + case ZF_MODE_WIDEN_S: widen_s(arg, i); break; +#if __riscv_xlen == 64 + case ZF_MODE_NARROW_D: narrow_d(arg, i); break; + case ZF_MODE_WIDEN_D: widen_d(arg, i); break; +#endif + case ZF_MODE_BOX: box(arg, i); break; + default: break; + } +} diff --git a/tests/regression/zfhmin/main.cpp b/tests/regression/zfhmin/main.cpp new file mode 100644 index 0000000000..93b99fe854 --- /dev/null +++ b/tests/regression/zfhmin/main.cpp @@ -0,0 +1,409 @@ +#include +#include +#include "common.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_random = 4096; +uint32_t frm_dyn = 3; +uint32_t phases = 0x3f; +uint32_t verbose = 0; +uint32_t num_flag_in = 160; + +vx_device_h dev = nullptr; +vx_queue_h q = nullptr; +vx_kernel_h kern = nullptr; +uint32_t xlen = 32; +int errors = 0; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:m:p:v:f:k:h")) != -1) { + switch (c) { + case 'n': num_random = std::atoi(optarg); break; + case 'p': phases = std::strtoul(optarg, nullptr, 0); break; + case 'v': verbose = std::atoi(optarg); break; + case 'f': num_flag_in = std::atoi(optarg); break; + case 'm': frm_dyn = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-n random cases] [-m dynamic frm] [-p phase mask] [-v dump count] [-f flag inputs] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +struct Buf { + vx_buffer_h h = nullptr; + uint64_t addr = 0; + uint64_t size = 0; + Buf(uint64_t sz) : size(sz ? sz : 8) { + CHECK(vx_buffer_create(dev, size, VX_MEM_READ_WRITE, &h)); + CHECK(vx_buffer_address(h, &addr)); + } + ~Buf() { vx_buffer_release(h); } + void write(const void* p) { CHECK(vx_enqueue_write(q, h, 0, p, size, 0, nullptr, nullptr)); } +}; + +void run(uint32_t mode, uint32_t count, Buf& src, Buf& dst, Buf* aux, void* out, uint64_t out_size) { + kernel_arg_t arg{}; + arg.mode = mode; + arg.count = count; + arg.frm = frm_dyn; + arg.src_addr = src.addr; + arg.dst_addr = dst.addr; + arg.aux_addr = aux ? aux->addr : 0; + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, out, dst.h, 0, out_size, 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); +} + +void read_aux(Buf& aux, void* out, uint64_t size) { + vx_event_h ev = nullptr; + CHECK(vx_enqueue_read(q, out, aux.h, 0, size, 0, nullptr, &ev)); + CHECK(vx_event_wait_value(ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(ev); +} + +void report(const char* what, uint64_t input, uint32_t rm, uint64_t expected, uint64_t actual) { + if (errors < 32) { + std::printf("*** %s in=0x%llx rm=%u expected=0x%llx actual=0x%llx\n", what, + (unsigned long long)input, rm, (unsigned long long)expected, (unsigned long long)actual); + } + ++errors; +} + +uint32_t rm_of(uint32_t k) { return (k < 5) ? k : frm_dyn; } + +int64_t sext16(uint64_t v) { return (int16_t)(uint16_t)v; } + +std::vector f32_inputs(std::mt19937& rng) { + std::vector v = { + 0x00000000, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, + 0x7f800001, 0xff800001, 0x7fa00000, 0x7fc12345, 0x7f7fffff, 0xff7fffff, + 0x00000001, 0x80000001, 0x007fffff, 0x00800000, + 0x477fe000, 0x477fefff, 0x477ff000, 0x477ff001, 0x477fffff, 0x47800000, + 0xc77fe000, 0xc77ff000, 0xc7800000, + 0x38800000, 0x387fffff, 0x387fe000, 0x387ff000, 0x33800000, 0x33000000, + 0x33000001, 0x32ffffff, 0x33400000, 0x33c00000, 0x2f800000, 0xb3000000, + 0xb3000001, 0x3f800000, 0x3f801000, 0x3f803000, 0x3f801001, 0x3f800fff, + 0xbf801000, 0xbf803000, 0x40490fdb, 0x3eaaaaab, + }; + const uint32_t low[] = {0, 1, 0xfff, 0x1000, 0x1001, 0x1fff, 0x2000, 0x3000, 0x7fffff, 0x7fefff}; + for (int e = -27; e <= 17; ++e) { + for (uint32_t l : low) { + for (int s = 0; s < 2; ++s) { + uint32_t hi = rng() & 0x7fe000; + v.push_back((uint32_t(s) << 31) | (uint32_t(e + 127) << 23) | ((hi | l) & 0x7fffff)); + } + } + } + for (uint32_t i = 0; i < num_random; ++i) { + uint32_t e = 127 - 30 + (rng() % 50); + v.push_back((rng() & 0x807fffff) | (e << 23)); + } + for (uint32_t i = 0; i < num_random / 4; ++i) { + v.push_back(rng()); + } + return v; +} + +std::vector f64_inputs(std::mt19937& rng) { + std::vector v = { + 0x0000000000000000ull, 0x8000000000000000ull, 0x7ff0000000000000ull, 0xfff0000000000000ull, + 0x7ff8000000000000ull, 0x7ff0000000000001ull, 0x7ff4000000000000ull, 0x7fefffffffffffffull, + 0x0000000000000001ull, 0x40effc0000000000ull, 0x40effdffffffffffull, 0x40effe0000000000ull, + 0x40effe0000000001ull, 0x40f0000000000000ull, 0x3f10000000000000ull, 0x3e70000000000000ull, + 0x3e60000000000000ull, 0x3e60000000000001ull, 0x3e5fffffffffffffull, 0x3ff0000000000000ull, + 0x3ff0020000000000ull, 0x3ff0020000000001ull, 0x3ff001ffffffffffull, 0xbff0060000000000ull, + }; + for (int e = -27; e <= 17; ++e) { + for (int k = 0; k < 8; ++k) { + uint64_t man = ((uint64_t(rng()) << 32) | rng()) & 0xfffffffffffffull; + if (k & 1) { + man = (man & ~0x3ffffffffffull) | 0x20000000000ull; + } + if (k & 2) { + man &= ~0x3ffffffffffull; + } + v.push_back((uint64_t(k & 4 ? 1 : 0) << 63) | (uint64_t(e + 1023) << 52) | man); + } + } + for (uint32_t i = 0; i < num_random; ++i) { + uint64_t e = 1023 - 30 + (rng() % 50); + uint64_t man = ((uint64_t(rng()) << 32) | rng()) & 0x800fffffffffffffull; + v.push_back(man | (e << 52)); + } + return v; +} + +void test_narrow_s(const std::vector& in) { + uint32_t n = in.size(); + Buf src(n * 4), dst(n * ZF_NUM_RM * 2), aux(n * 8); + src.write(in.data()); + std::vector out(n * ZF_NUM_RM); + std::vector mvx(n); + run(ZF_MODE_NARROW_S, n, src, dst, &aux, out.data(), dst.size); + read_aux(aux, mvx.data(), aux.size); + for (uint32_t i = 0; i < n && i < verbose; ++i) { + std::printf("in=0x%08x", in[i]); + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { + std::printf(" %04x/%04x", out[i * ZF_NUM_RM + k], rv_ftoh_s(in[i], rm_of(k), nullptr)); + } + std::printf("\n"); + } + for (uint32_t i = 0; i < n; ++i) { + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { + uint16_t ref = rv_ftoh_s(in[i], rm_of(k), nullptr); + if (out[i * ZF_NUM_RM + k] != ref) { + report("fcvt.h.s", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } + } + int64_t ref = sext16(rv_ftoh_s(in[i], 0, nullptr)); + if (mvx[i] != ref) { + report("fmv.x.h", in[i], 0, ref, mvx[i]); + } + } + std::printf("fcvt.h.s: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); +} + +void test_widen_s() { + uint32_t n = 65536; + std::vector in(n); + for (uint32_t i = 0; i < n; ++i) { + in[i] = i; + } + Buf src(n * 2), dst(n * 4); + src.write(in.data()); + std::vector out(n); + run(ZF_MODE_WIDEN_S, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + uint32_t ref = rv_htof_s(in[i], 0, nullptr); + if (out[i] != ref) { + report("fcvt.s.h", in[i], 0, ref, out[i]); + } + } + std::printf("fcvt.s.h: all %u half inputs\n", n); +} + +void test_narrow_d(const std::vector& in) { + uint32_t n = in.size(); + Buf src(n * 8), dst(n * ZF_NUM_RM * 2); + src.write(in.data()); + std::vector out(n * ZF_NUM_RM); + run(ZF_MODE_NARROW_D, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + for (uint32_t k = 0; k < ZF_NUM_RM; ++k) { + uint16_t ref = rv_dtoh(in[i], rm_of(k), nullptr); + if (out[i * ZF_NUM_RM + k] != ref) { + report("fcvt.h.d", in[i], rm_of(k), ref, out[i * ZF_NUM_RM + k]); + } + } + } + std::printf("fcvt.h.d: %u inputs x %u rounding modes\n", n, ZF_NUM_RM); +} + +void test_widen_d() { + uint32_t n = 65536; + std::vector in(n); + for (uint32_t i = 0; i < n; ++i) { + in[i] = i; + } + Buf src(n * 2), dst(n * 8); + src.write(in.data()); + std::vector out(n); + run(ZF_MODE_WIDEN_D, n, src, dst, nullptr, out.data(), dst.size); + for (uint32_t i = 0; i < n; ++i) { + uint64_t ref = rv_htod(in[i], 0, nullptr); + if (out[i] != ref) { + report("fcvt.d.h", in[i], 0, ref, out[i]); + } + } + std::printf("fcvt.d.h: all %u half inputs\n", n); +} + +void test_flags(const std::vector& f32, const std::vector& f64) { + std::vector cases; + for (size_t i = 0; i < num_flag_in && i < f32.size(); ++i) { + for (uint32_t rm = 0; rm < 5; ++rm) { + cases.push_back({ZF_OP_H_S, rm, f32[i]}); + } + } + const uint16_t halves[] = {0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0x7bff, 0x7c00, + 0xfc00, 0x7e00, 0x7c01, 0x7d00, 0xfe00, 0xfd55, 0x7fff, 0x1234}; + for (uint16_t h : halves) { + cases.push_back({ZF_OP_S_H, 0, h}); + } + if (xlen == 64) { + for (size_t i = 0; i < 64 && i < f64.size(); ++i) { + for (uint32_t rm = 0; rm < 5; ++rm) { + cases.push_back({ZF_OP_H_D, rm, f64[i]}); + } + } + for (uint16_t h : halves) { + cases.push_back({ZF_OP_D_H, 0, h}); + } + } + uint32_t n = cases.size(); + Buf src(n * sizeof(zf_flag_case_t)), dst(n * sizeof(zf_flag_out_t)); + src.write(cases.data()); + std::vector out(n); + run(ZF_MODE_FLAGS, n, src, dst, nullptr, out.data(), dst.size); + static const char* names[] = {"fcvt.h.s", "fcvt.s.h", "fcvt.h.d", "fcvt.d.h"}; + for (uint32_t i = 0; i < n; ++i) { + const auto& c = cases[i]; + uint32_t fl = 0; + uint64_t ref = 0; + switch (c.op) { + case ZF_OP_H_S: ref = rv_ftoh_s(uint32_t(c.bits), c.rm, &fl); break; + case ZF_OP_S_H: ref = rv_htof_s(uint16_t(c.bits), c.rm, &fl); break; + case ZF_OP_H_D: ref = rv_dtoh(c.bits, c.rm, &fl); break; + case ZF_OP_D_H: ref = rv_htod(uint16_t(c.bits), c.rm, &fl); break; + } + if (out[i].result != ref) { + report(names[c.op], c.bits, c.rm, ref, out[i].result); + } + if (out[i].fflags != fl) { + char what[32]; + std::snprintf(what, sizeof(what), "%s fflags", names[c.op]); + report(what, c.bits, c.rm, fl, out[i].fflags); + } + } + std::printf("fflags: %u single-lane cases\n", n); +} + +void test_box(std::mt19937& rng) { + uint32_t n = 1024; + std::vector h(n); + std::vector g(n); + for (uint32_t i = 0; i < n; ++i) { + h[i] = rng(); + g[i] = rng(); + if (i % 4 == 0) { + g[i] |= 0xffff0000; + } + if (i % 8 == 1) { + g[i] &= 0x0000ffff; + } + } + Buf src(n * 2), aux(n * 4), dst(n * ZF_BOX_OUTS * 8); + src.write(h.data()); + aux.write(g.data()); + std::vector out(n * ZF_BOX_OUTS); + run(ZF_MODE_BOX, n, src, dst, &aux, out.data(), dst.size); + uint64_t box = (xlen == 64) ? 0xffffffffffff0000ull : 0xffff0000ull; + for (uint32_t i = 0; i < n; ++i) { + const uint64_t* o = &out[i * ZF_BOX_OUTS]; + uint64_t r0 = box | h[i]; + uint64_t r1 = box | (g[i] & 0xffff); + uint64_t r2 = (uint64_t)sext16(h[i]); + uint64_t r3 = ((g[i] >> 16) == 0xffff) ? rv_htof_s(uint16_t(g[i]), 0, nullptr) : 0x7fc00000u; + uint64_t r4 = (uint64_t)sext16(g[i]); + if (o[0] != r0) { + report("flh box", h[i], 0, r0, o[0]); + } + if (o[1] != r1) { + report("fmv.h.x box", g[i], 0, r1, o[1]); + } + if (o[2] != r2) { + report("flh+fmv.x.h", h[i], 0, r2, o[2]); + } + if (o[3] != r3) { + report("unboxed fcvt.s.h", g[i], 0, r3, o[3]); + } + if (o[4] != r4) { + report("fmv.x.h unboxed", g[i], 0, r4, o[4]); + } + } + std::printf("boxing: %u cases\n", n); +} + +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + std::mt19937 rng(42); + + CHECK(vx_device_open(0, &dev)); + uint64_t isa = 0; + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + if (!(isa & VX_ISA_EXT_ZFHMIN)) { + std::cout << "Error: device does not support Zfhmin (VX_CFG_EXT_ZFHMIN_ENABLE)" << std::endl; + vx_device_release(dev); + return 1; + } + xlen = VX_ISA_ARCH(isa); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + CHECK(vx_queue_create(dev, &qi, &q)); + vx_module_h mod = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + std::printf("zfhmin: xlen=%u dynamic frm=%u\n", xlen, frm_dyn); + auto f32 = f32_inputs(rng); + auto f64 = f64_inputs(rng); + if (phases & 0x01) { + test_narrow_s(f32); + } + if (phases & 0x02) { + test_widen_s(); + } + if (xlen == 64) { + if (phases & 0x04) { + test_narrow_d(f64); + } + if (phases & 0x08) { + test_widen_d(); + } + } + if (phases & 0x10) { + test_flags(f32, f64); + } + if (phases & 0x20) { + test_box(rng); + } + + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +}