diff --git a/.gitignore b/.gitignore index dbcea3d6e3..136b7f2186 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ hw/syn/libs/asap7/verilog/ hw/syn/libs/asap7/metadata/ hw/syn/libs/asap7/.installed_* hw/syn/libs/asap7/install.sh + +# Built by third_party/Makefile +third_party/ramulator_ext/*.o +third_party/ramulator_ext/*.so diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7cb17ba7..b384f6a82b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to Vortex are documented here. The format is based on follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, `TOOLCHAIN_REV`, `GEM5_REV`). +## [Unreleased] + +### Added + +- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow + builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. + See [docs/simulation.md](docs/simulation.md#hardware-profiles). + +### Changed + +- **RV64 portability.** Device index math, DXA/OM addresses, pack-load strides, RTU scene/payload pointers and the ray log carry full + width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. +- **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and + Verilator warnings are fatal again, with the RTL and host code fixed to build clean. +- **DRAM model redesign.** simx, rtlsim, opaesim and xrtsim model one independent Ramulator controller and device per memory bank + (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel + responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine + keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. + ## [3.0] — 2026-10-07 The 3.0 release introduces a fixed-function graphics stack (rasterizer, texture units, and output mergers), tensor core structured sparsity (2:4), warpgroup-level matrix multiplication (WGMMA), global-to-local data transfer acceleration (DXA), a new hardware kernel scheduler (KMU) and Command Processor (CP) architecture, a new asynchronous runtime API (`vortex2.h`), asynchronous barriers with arrive/wait/event semantics, compressed instruction set (RVC) support, hardware atomics, an MMU/SV32 virtual memory stack, a Mesa/lavapipe Vulkan backend (`vortexpipe`), HIP via chipStar, gem5 integration, a SimX v3 TLM architecture with fixed-size handshake channels, productized Synopsys and Yosys ASIC synthesis flows, and a refreshed toolchain (LLVM 20, POCL 7.0). Build and configuration infrastructure was reworked: TOML-driven HW configuration ([VX_config.toml](VX_config.toml) + [VX_types.toml](VX_types.toml)) decoupling SimX/runtime from the RTL source tree, a `VX_CFG_` macro namespace that resolves toolchain preprocessor collisions, retirement of the global `toolchain_env.sh` to enable parallel multi-version Vortex worktrees on the same shell, consolidation of `kernel/`/`runtime/` under a shared `sw/` root, a single-source [VERSION](VERSION) file driving CI toolchain pinning, Perfetto trace export ([ci/perfetto.py](ci/perfetto.py)), and new top-level [AGENTS.md](AGENTS.md) + [CONTRIBUTING.md](CONTRIBUTING.md) for AI-agent and contributor workflows. diff --git a/VX_config.toml b/VX_config.toml index a3336551a8..f51186cdec 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -79,6 +79,17 @@ VX_CFG_PLATFORM_MEMORY_DATA_SIZE = 64 VX_CFG_PLATFORM_MEMORY_INTERLEAVE = 1 VX_CFG_PLATFORM_MEMORY_PEAK_BW = 460000 # 460 GB/s VX_CFG_PLATFORM_CLOCK_RATE = 400 +# DRAM the simulators model behind each memory bank (one channel per bank), +# as a data sheet gives it: the JEDEC standard, its data rate (MT/s per pin), +# the fixed read latency added for the PHY, controller and interconnect (core +# cycles), and the requests a channel holds in flight (0 = unbounded). +# Simulation-only: the RTL never reads them. The default clocks the DRAM with +# the core: 14 cycles for an idle open-row read, 26 for a row conflict, one +# 64 B block per cycle at peak; profiles set their own. +VX_CFG_PLATFORM_MEMORY_TYPE = "HBM2" +VX_CFG_PLATFORM_MEMORY_DATA_RATE = "expr: 2 * $VX_CFG_PLATFORM_CLOCK_RATE" +VX_CFG_PLATFORM_MEMORY_LATENCY = 0 +VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE = 0 [alu] VX_CFG_NUM_ALU_LANES = "expr: $VX_CFG_SIMD_WIDTH" @@ -456,6 +467,7 @@ VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] +VX_CFG_PLATFORM_MEMORY_TYPE = ["HBM2", "HBM3", "GDDR6"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/VX_types.toml b/VX_types.toml index dc0f368b2b..7143523976 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -442,6 +442,7 @@ VX_RT_STATUS = 26 # per-lane VX_RT_STS_*; WAIT completes on # Every slot is written by the RTU and read by the shader; nothing else writes the # window. The payload pointer rides the arm doorbell (warp-uniform) and the # hitAttribute rides the CONTINUE, so both reach the RTU without a window write. +# On RV64 the payload pointer's upper word follows in slot VX_RT_PAYLOAD_PTR_LO + 1. VX_RT_PAYLOAD_PTR_LO = 27 # the tracing shader's payload; the callbacks read it VX_RT_HIT_ATTR_0 = 29 # user hitAttributeEXT (an IS shader returns it) diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 5e226eba16..77e8c4785a 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 568490, + "cycles": 563524, "instrs": 137486 }, "app": "raycast", "args": "", - "config_hash": "ee3a227211fa9942", + "config_hash": "396cee9df6025cdb", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1495440, + "cycles": 1498642, "instrs": 259228 }, "app": "raycast", "args": "", - "config_hash": "c8b680eabeb18655", + "config_hash": "262f24032ef14b58", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 781696, + "cycles": 787850, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "58eb3cc06964207a", + "config_hash": "ad2b37492d761e8f", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2211340, + "cycles": 2239426, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "aa6f26c7306417a8", + "config_hash": "606d011dc6f612d3", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1823521, + "cycles": 1838654, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "b390d957eec8220f", + "config_hash": "9390754d3719fca3", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 6055079, + "cycles": 5939731, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "13d03df76d55105e", + "config_hash": "740fd84fa2938103", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1297311, + "cycles": 1290130, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "68638b6685801699", + "config_hash": "2e1c44e05f9ac34d", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4210203, + "cycles": 4219466, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "837240be3049d62e", + "config_hash": "024adb49ed96bdd3", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 633593, - "instrs": 79120 + "cycles": 685107, + "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "c6a3720118661bf7", + "config_hash": "a3d9df9695acbb95", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 1060219, - "instrs": 316432 + "cycles": 872926, + "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "db6bb5af90ecc1a5", + "config_hash": "0e8990b25e64ad7f", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1274656, + "cycles": 1272929, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "1c0e7742deb14093", + "config_hash": "9e0c1c9767c760ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860631, + "cycles": 4860309, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "3797eecae3200315", + "config_hash": "eb5fed4bfce19d76", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2388738, + "cycles": 2306960, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "aa768c2a9c65b092", + "config_hash": "6925a5138be930ea", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4177542, + "cycles": 4180969, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "5400d882a6c96ac1", + "config_hash": "b83a1e3735f79d39", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 75714, - "instrs": 24592 + "cycles": 100640, + "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "98b8bdc5e546ea62", + "config_hash": "ef0c2548e7c5bf86", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 293078, - "instrs": 98320 + "cycles": 367765, + "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "1510b8438164b252", + "config_hash": "ce1eb79984a98552", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index d864f76b96..bf1e1a010c 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,122 +1,122 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3411, + "cycles": 3381, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "cbcd10be884ceab7", + "config_hash": "6a815f855f6e0d05", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 2096, + "cycles": 1994, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a1ec1b81b5b3baf", + "config_hash": "626c5f6244584b1f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt4:rtlsim": { "32": { - "cycles": 6844, + "cycles": 6827, "instrs": 1792 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "3a888067fcc449d6", + "config_hash": "7c32a11d123bfb13", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14333410, + "cycles": 14022240, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "14050e4dcef65266", + "config_hash": "822d0ca194266a27", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14345910, + "cycles": 14063502, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "a55b900d67aa3657", + "config_hash": "87ea544e77719254", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3978026, + "cycles": 3941267, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "36f5886fdc97e040", + "config_hash": "e55b61598f5f10ea", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14632454, + "cycles": 14352313, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "9ccf890d5ac45a62", + "config_hash": "21da2f2737bf03dd", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 98435, + "cycles": 82160, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "ee7678a7e7e9b79b", + "config_hash": "20304652a1bfb94c", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 400679, + "cycles": 360010, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "c79015e4caec7b93", + "config_hash": "bf4520d008c07dc9", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 39761, + "cycles": 36021, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "5a35710ef3aec3a6", + "config_hash": "2a4e017d98740636", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 68208, + "cycles": 59789, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "d12fad3fd43e61c6", + "config_hash": "f05069d7cf0ee3ff", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 32c723239d..b5ee86bfc2 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 531740, - "instrs": 204102 + "cycles": 530195, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "c8c2af44277e2cb0", + "config_hash": "1e2937b20579ad3f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 145662, - "instrs": 52516 + "cycles": 143620, + "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ee0410ff09e7ea58", + "config_hash": "edc924bc4f246971", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 531359, - "instrs": 204102 + "cycles": 529393, + "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "00c7939a6d958e0e", + "config_hash": "d682cc8c6f99c06b", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 191808, - "instrs": 69648 + "cycles": 189231, + "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "60103e7b6eb2c625", + "config_hash": "46a66220c1d9956b", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 754454, - "instrs": 278544 + "cycles": 740701, + "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "ee88907a281d364a", + "config_hash": "a7554f6fa88474dc", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14959, + "cycles": 14785, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "3ba867af954e5cbf", + "config_hash": "e2e1879d7d6f9aaf", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55633, + "cycles": 55565, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "859b3c4bd2208b92", + "config_hash": "6e48f6b249533cc5", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 34994, + "cycles": 35441, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "1c38a250b6b85273", + "config_hash": "c8c6b30537e93e2f", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136738, + "cycles": 136487, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "226df03058dfb436", + "config_hash": "a4fc4d97e6e80586", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index 86b562e3b1..1fb0ab7f7b 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 84873, + "cycles": 85045, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "c71b720ea7aa7e72", + "config_hash": "9a4f28d2555289b5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53310, + "cycles": 53434, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "3049a9d721044cde", + "config_hash": "aef578263250d585", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159542, + "cycles": 159655, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "14ae0ed831d1cd4a", + "config_hash": "c7b45fe5c1828bc5", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 3249cc1b16..3552c4620b 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 195794, + "cycles": 197465, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "6a095ed165469a30", + "config_hash": "da0a50e46103b7dc", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 44583, + "cycles": 42404, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "0fe2a8e2a75fcc10", + "config_hash": "2d12f1a0058fcc3b", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221331, + "cycles": 221592, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "b2ef81f2fa1e6a43", + "config_hash": "fce886092e60e0eb", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 251208, + "cycles": 256743, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "5534ddd0051a5c15", + "config_hash": "f0558a3b3dfde51f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 592658, + "cycles": 590798, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "421a4ab52f0f072a", + "config_hash": "24fd7a811fabb8c4", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index c9b137227e..1dc1b78f41 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 77127, + "cycles": 76393, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "719fe3a7ce81115d", + "config_hash": "3cfc301023342a7d", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index e8e15a7ab2..2c4f92004c 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 187290, + "cycles": 183094, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "acf4fa4675243f0f", + "config_hash": "90defe3d20b6f739", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 6ea2c1d004..c7f771c72f 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 157272, + "cycles": 158181, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "7e8b9759dc36dba0", + "config_hash": "ba81956fe68e94b6", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 841657, + "cycles": 834873, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "c5a71c297fc4dafe", + "config_hash": "6ee14f25ec573117", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 401322, + "cycles": 400023, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "ab1451598dfbc872", + "config_hash": "a9ea3d8bd83a7eed", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/blackbox.sh b/ci/blackbox.sh index f4072e9ada..21215167a0 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -19,7 +19,7 @@ ROOT_DIR=$SCRIPT_DIR/.. show_usage() { echo "Vortex BlackBox Test Driver v1.0" - echo "Usage: $0 [[--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" + echo "Usage: $0 [[--profile=#name] [--clusters=#n] [--cores=#n] [--warps=#n] [--threads=#n] [--l2cache] [--l3cache] [[--driver=#name] [--target=#name] [--app=#app] [--args=#args] [--debug=#level] [--scope] [--vcd] [--saif] [--perf=#class] [--vcd_file=#file] [--saif_file=#file] [--log=logfile] [--nohup] [--help]]" } show_help() @@ -66,6 +66,7 @@ parse_args() { for i in "$@"; do case $i in --driver=*) DRIVER=${i#*=} ;; + --profile=*) export PROFILE=${i#*=} ;; --target=*) TARGET=${i#*=} ;; --app=*) APP=${i#*=} ;; --clusters=*) CONFIGS=$(add_option "$CONFIGS" "-DVX_CFG_NUM_CLUSTERS=${i#*=}") ;; diff --git a/ci/gen_config.py b/ci/gen_config.py index 1dd4c83ced..316c8d0a97 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -39,6 +39,9 @@ definitions that can be overridden from -D flags. - Resolved mode (-r/--resolved): fully evaluates expressions and emits resolved constants (always enabled for cflags). + - Hardware profiles (--profile): a named machine in profiles/.json layered + between the TOML defaults and explicit -D flags. --merge-configs prints the + caller's CONFIGS with the profile folded in, one -D per macro. """ from __future__ import annotations @@ -46,6 +49,7 @@ import argparse import ast import copy +import json import os import re import shlex @@ -309,6 +313,13 @@ class EnumSpec: values: List[Any] +def _enum_of_flag(name: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The enum a bare -D_ flag selects. A value may itself contain + # '_', so the enum name is matched as a prefix instead of splitting the flag. + owners = [e for e in enums if name.startswith(e + "_")] + return max(owners, key=len) if owners else None + + def _load_enums(toml_data: Dict[str, Any]) -> Dict[str, EnumSpec]: enums = toml_data.get("enum", None) if enums is None: @@ -450,10 +461,10 @@ def set_enum(name: str, val: Any) -> None: set_enum(d.name, _scalar(d.value)) continue - if "_" in d.name and d.value is None: - base, suffix = d.name.rsplit("_", 1) - if base in enums: - set_enum(base, _scalar(suffix)) + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: + set_enum(base, _scalar(d.name[len(base) + 1:])) overrides[d.name] = True explicit.add(d.name) continue @@ -468,6 +479,124 @@ def set_enum(name: str, val: Any) -> None: return overrides, explicit +# ----------------------------- +# Hardware profiles +# ----------------------------- + +PROFILE_SECTIONS = ("core", "chip", "platform") +PROFILE_FIELDS = {"name", "extends", "description", "reference", "sources"} | set(PROFILE_SECTIONS) + + +@dataclass +class Profile: + name: str + chain: List[str] # base first + values: Dict[str, Any] # VX_CFG_* knob -> literal value + sections: Dict[str, Set[str]] # section -> knobs it sets + reference: Dict[str, Any] + sources: Any + + +def _profile_path(name: str, profiles_dir: str) -> str: + if name.endswith(".json") and os.path.isfile(name): + return name + return os.path.join(profiles_dir, f"{name}.json") + + +def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> Profile: + if name in _seen: + raise ValueError(f"Profile inheritance cycle: {' -> '.join(_seen + (name,))}") + path = _profile_path(name, profiles_dir) + if not os.path.isfile(path): + raise ValueError(f"Unknown profile '{name}' (no {path})") + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + unknown = set(data.keys()) - PROFILE_FIELDS + if unknown: + raise ValueError(f"Profile '{name}': unknown field(s) {sorted(unknown)} (allowed: {sorted(PROFILE_FIELDS)})") + + if "extends" in data: + base = load_profile(data["extends"], profiles_dir, _seen + (name,)) + else: + base = Profile(name="", chain=[], values={}, sections={s: set() for s in PROFILE_SECTIONS}, + reference={}, sources=None) + + values = dict(base.values) + sections = {s: set(base.sections[s]) for s in PROFILE_SECTIONS} + for sec in PROFILE_SECTIONS: + for k, v in data.get(sec, {}).items(): + for other in PROFILE_SECTIONS: + sections[other].discard(k) + values[k] = v + sections[sec].add(k) + reference = dict(base.reference) + reference.update(data.get("reference", {})) + sources = data.get("sources", base.sources) + return Profile(name=data.get("name", name), chain=base.chain + [name], values=values, + sections=sections, reference=reference, sources=sources) + + +def validate_profile(prof: Profile, toml_defs: Dict[str, Any], enums: Dict[str, EnumSpec], + read_only: Set[str]) -> None: + if not prof.sources: + raise ValueError(f"Profile '{prof.name}': 'sources' is required (where every value comes from)") + for k, v in prof.values.items(): + if k in read_only: + raise ValueError(f"Profile '{prof.name}': {k} is a read-only [[param]]/[[builtin]] symbol") + if k not in toml_defs and k not in enums: + raise ValueError(f"Profile '{prof.name}': unknown knob {k} (not in VX_config.toml)") + if not _has_public_scope(k): + raise ValueError(f"Profile '{prof.name}': {k} is a private TOML local") + if k in enums: + if v not in enums[k].values: + raise ValueError(f"Profile '{prof.name}': invalid {k}={v!r} (allowed: {enums[k].values})") + elif k.endswith("_ENABLE"): + if not isinstance(v, bool): + raise ValueError(f"Profile '{prof.name}': {k} must be true or false") + elif isinstance(v, bool) or not isinstance(v, (int, str)): + raise ValueError(f"Profile '{prof.name}': {k} must be an integer or a string literal") + elif isinstance(v, str) and _is_expr_string(v): + raise ValueError(f"Profile '{prof.name}': {k} must be a literal; expressions belong in VX_config.toml") + + +def profile_defines(prof: Profile) -> List[str]: + toks: List[str] = [f"-DVX_PROFILE={prof.name}"] + for k in sorted(prof.values): + v = prof.values[k] + if isinstance(v, bool): + toks.append(f"-D{k}" if v else f"-D{k[:-len('_ENABLE')]}_DISABLE") + else: + toks.append(f"-D{k}={v}") + return toks + + +def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: + # The knob a -D token sets, so X_DISABLE and X_ENABLE, or X_ and X=, + # are recognised as the same macro. + defs = _parse_defines([tok]) + if not defs: + return None + d = defs[0] + if d.name.endswith("_DISABLE"): + return d.name[:-len("_DISABLE")] + "_ENABLE" + if d.name in enums: + return d.name + if d.value is None: + base = _enum_of_flag(d.name, enums) + if base is not None: + return base + return d.name + + +def merge_configs(prof: Profile, cflags: str, enums: Dict[str, EnumSpec]) -> str: + explicit = shlex.split(cflags) if cflags else [] + given = {_canonical_define(t, enums) for t in explicit} + given.discard(None) + merged = [t for t in profile_defines(prof) if _canonical_define(t, enums) not in given] + # The caller's flags pass through verbatim, quoting included. + return " ".join(merged + ([cflags.strip()] if cflags.strip() else [])) + + # ----------------------------- # Dialect helpers # ----------------------------- @@ -1489,6 +1618,11 @@ def main(argv: List[str]) -> int: ap.add_argument("--output", "-o", default=None) ap.add_argument("--resolved", "-r", action="store_true", help="Emit resolved constants for cpp/verilog (cflags is always resolved)") + ap.add_argument("--profile", default="", + help="hardware profile: profiles/.json next to the config, or a .json path") + ap.add_argument("--profiles-dir", default=None, help="profile directory (default: /profiles)") + ap.add_argument("--merge-configs", action="store_true", + help="print --cflags with the profile folded in (explicit -D flags win) and exit") args, unknown = ap.parse_known_args(argv) hex_meta = _scan_hex_literals(args.config) @@ -1501,6 +1635,26 @@ def main(argv: List[str]) -> int: toml_defs, layout = _flatten_with_layout(toml_data) + prof: Optional[Profile] = None + if args.profile and args.profile != "none": + profiles_dir = args.profiles_dir or os.path.join(os.path.dirname(os.path.abspath(args.config)), "profiles") + prof = load_profile(args.profile, profiles_dir) + validate_profile(prof, toml_defs, enums, params_set | set(builtins.keys())) + + if args.merge_configs: + out = merge_configs(prof, args.cflags, enums) if prof else args.cflags + sys.stdout.write(out.strip() + "\n") + return 0 + + if prof is not None: + if args.format == "cflags" or args.resolved: + # Resolved outputs see the profile as the lowest-priority overrides. + args.cflags = merge_configs(prof, args.cflags, enums) + else: + # Unresolved headers carry the profile as their overridable defaults. + for k, v in prof.values.items(): + toml_defs[k] = v + defs: List[Define] = [] defs += _parse_defines_from_cflags(args.cflags) defs += _parse_defines(unknown) diff --git a/ci/regression.sh.in b/ci/regression.sh.in index 70165bfa08..774434c5d3 100755 --- a/ci/regression.sh.in +++ b/ci/regression.sh.in @@ -118,7 +118,7 @@ gem5() echo "begin gem5 tests..." if [ -z "$GEM5_HOME" ]; then - GEM5_HOME=$HOME/tools/gem5 + GEM5_HOME=${TOOLDIR:-@TOOLDIR@}/gem5 fi if [ ! -x "$GEM5_HOME/build/X86/gem5.opt" ]; then echo "error: $GEM5_HOME/build/X86/gem5.opt not found — run ci/gem5_install.sh first" diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index 704debca65..c337b03306 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -548,6 +548,11 @@ tests: - {id: gfx_pipeline_tex, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_tex, target: "run-{driver}"} - {id: gfx_pipeline_om, via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_om, target: "run-{driver}"} - {id: gfx_pipeline_raster,via: make-run, drivers: [simx], dir: tests/graphics/gfx_pipeline_raster,target: "run-{driver}"} +# The front-end setup, bin-sort and full pipeline kernels, each checked byte for +# byte against the host reference built from the same sw/gfx and sw/common headers. +- {id: gfx_setup_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_setup_kernel, target: "run-{driver}"} +- {id: gfx_binsort_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_binsort_kernel, target: "run-{driver}"} +- {id: gfx_pipeline_kernel, via: make-run, drivers: [simx], dir: tests/regression/gfx_pipeline_kernel, target: "run-{driver}"} # ── FF/SW routing matrix ───────────────────────────────────────────────────── # Each fixed-function unit dropped from the build (NO_TEX / NO_OM / NO_RASTER); diff --git a/ci/testcases/raytracing.yaml b/ci/testcases/raytracing.yaml index 6ccbc74045..734ef894cf 100644 --- a/ci/testcases/raytracing.yaml +++ b/ci/testcases/raytracing.yaml @@ -1,8 +1,7 @@ # Test cases for `raytracing` (RTU), ported from regression.sh.in raytracing(). # -# simx is the only backend that validates the RTU end-to-end today; rtlsim / -# opae / xrt RTU coverage lands when the RTL side comes online. xlen is pinned to -# 32 (the RTU SimX-parity reference build); 64-bit RTU coverage is a follow-on. +# xlen defaults to 32 (the RTU SimX-parity reference build); rt_suite-x64 runs +# the suite on RV64. category: raytracing defaults: xlen: @@ -30,6 +29,20 @@ tests: target: run-{driver} configs: -DVX_CFG_EXT_RTU_ENABLE +# The same suite on RV64. There rt_smoke_chs puts its scene and payload above +# 4 GiB, so a trace that narrows either pointer misses or loses its payload, and +# the FP hit-window reads must come back NaN-boxed. +- id: rt_suite-x64 + via: make-run + xlen: + - 64 + drivers: + - simx + - rtlsim + dir: tests/raytracing + target: run-{driver} + configs: -DVX_CFG_EXT_RTU_ENABLE + # Multi-core RTU coverage: same canary kernel (rt_smoke) under two 4-core # topologies so a topology-only break is a single failure. -n64 = full device # capacity (cores x warps x threads); 64 rays in flight through the cluster's diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 738eb3851f..2c4d2b7c2b 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -170,6 +170,17 @@ tests: target: run-{driver} vars: OPTS: -t4 -n4 -p1 +# The heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP): every buffer +# translates through the TLBs' megapage path at its identity VA. +- id: vm-megapage-heap + via: make-run + drivers: + - simx + - rtlsim + dir: tests/regression/sgemm + target: run-{driver} + env: + VORTEX_VM_MEGAPAGE_HEAP: 1 - id: tlb-concurrent via: make-run drivers: diff --git a/config.mk.in b/config.mk.in index 8fe66bbd0d..1dc8f4efbd 100644 --- a/config.mk.in +++ b/config.mk.in @@ -55,3 +55,14 @@ SIM_COMMON_DIR ?= $(VORTEX_HOME)/sim/common # V80 runtime install prefix; only consulted by the aved backend when it # targets real hardware or the vendor simulator. VRT_HOME ?= /usr/local + +# Hardware profile (profiles/.json), folded into CONFIGS once here so every +# flow that includes config.mk builds the same machine. Explicit -D flags in +# CONFIGS win over the profile. +PROFILE ?= @PROFILE@ +ifneq ($(strip $(PROFILE)),) +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +ifneq ($(.SHELLSTATUS),0) +$(error invalid hardware profile '$(PROFILE)') +endif +endif diff --git a/configure b/configure index f9340d12ee..c5e1e21147 100755 --- a/configure +++ b/configure @@ -77,7 +77,7 @@ copy_files() { continue fi mkdir -p "$dest_dir" - sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g" "$file" > "$dest_file" + sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g" "$file" > "$dest_file" # apply permissions to bash scripts read -r firstline < "$dest_file" if [[ "$firstline" =~ ^#!.*bash ]]; then @@ -140,6 +140,7 @@ default_xlen=32 default_tooldir=$HOME/tools default_osversion=$(detect_osversion) default_prefix=$CURRENT_DIR/install +default_profile= # load default configuration parameters from existing config.mk if [ -f "config.mk" ]; then @@ -152,6 +153,7 @@ if [ -f "config.mk" ]; then TOOLDIR\ ?*) default_tooldir=${value//\?=/} ;; OSVERSION\ ?*) default_osversion=${value//\?=/} ;; PREFIX\ ?*) default_prefix=${value//\?=/} ;; + PROFILE\ ?*) default_profile=${value//\?=/} ;; esac done < config.mk fi @@ -161,6 +163,7 @@ XLEN=${XLEN:=$default_xlen} TOOLDIR=${TOOLDIR:=$default_tooldir} OSVERSION=${OSVERSION:=$default_osversion} PREFIX=${PREFIX:=$default_prefix} +PROFILE=${PROFILE-$default_profile} # parse command line arguments usage() { @@ -169,6 +172,7 @@ usage() { echo " --tooldir= Set the TOOLDIR path (default: $HOME/tools)" echo " --osversion= Set the OS Version (default: $(detect_osversion))" echo " --prefix=, --installdir= Set installation directory" + echo " --profile= Default hardware profile, profiles/.json (default: none)" exit 1 } while [[ "$#" -gt 0 ]]; do @@ -177,12 +181,19 @@ while [[ "$#" -gt 0 ]]; do --tooldir=*) TOOLDIR="${1#*=}" ;; --osversion=*) OSVERSION="${1#*=}" ;; --prefix=*|--installdir=*) PREFIX="${1#*=}" ;; + --profile=*) PROFILE="${1#*=}" ;; -h|--help) usage ;; *) echo "Unknown parameter passed: $1"; usage ;; esac shift done +# check the profile before anything is generated from it +if [ -n "$PROFILE" ] && [ ! -f "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/profiles/$PROFILE.json" ]; then + echo "Error: unknown profile '$PROFILE' (no profiles/$PROFILE.json)" + exit -1 +fi + # check OS if [ "$OSVERSION" == "unsupported" ]; then echo "Error: Unsupported OS." @@ -202,7 +213,7 @@ source "$SOURCE_DIR/VERSION" # stamp file tracks config parameter changes; touched only when params change CONFIG_STAMP="$CURRENT_DIR/.config.stamp" -CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" +CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX PROFILE=$PROFILE SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" if [ ! -f "$CONFIG_STAMP" ] || [ "$(cat "$CONFIG_STAMP" 2>/dev/null)" != "$CONFIG_SIG" ]; then echo "$CONFIG_SIG" > "$CONFIG_STAMP" fi @@ -238,6 +249,7 @@ for toml in "$SOURCE_DIR"/*.toml; do extra="" case "$name" in VX_types) extra="--resolved" ;; + VX_config) [ -n "$PROFILE" ] && extra="--profile=$PROFILE" ;; esac gen_header "$toml" "$CURRENT_DIR/hw/$name.vh" verilog "$extra" gen_header "$toml" "$CURRENT_DIR/sw/$name.h" cpp "$extra" diff --git a/docs/codebase.md b/docs/codebase.md index e355ad3045..9e68135e0f 100644 --- a/docs/codebase.md +++ b/docs/codebase.md @@ -5,6 +5,8 @@ The directory/file layout of the Vortex codebase is as follows: - `VX_config.toml` / `VX_types.toml`: hardware configuration system — the single source of truth for all build-time parameters (see the [build configuration system](designs/build_configuration_system.md)). +- `profiles`: hardware profiles — named machines (`a100`, `rtx5080`, `tinygpu`, ...) + layered over `VX_config.toml` (see [Hardware Profiles](simulation.md#hardware-profiles)) - `ci`: continuous integration scripts - `testcases`: declarative test catalog (YAML), executed by the pytest harness - `baselines/perf`: golden performance baselines for the perf-regression gate @@ -74,6 +76,7 @@ The directory/file layout of the Vortex codebase is as follows: - `runtime`: driver API tests - `mpi`: multi-process tests - `third_party`: external library submodules (cvfpu, softfloat, hardfloat, - ramulator, cocogfx) + ramulator, cocogfx); `ramulator_ext` holds the build of the GDDR6 device + model Ramulator ships but does not build - `perf`: performance analysis resources - `miscs`: miscellaneous resources diff --git a/docs/designs/build_configuration_system.md b/docs/designs/build_configuration_system.md index c231a03bbe..ccc4a9a3bb 100644 --- a/docs/designs/build_configuration_system.md +++ b/docs/designs/build_configuration_system.md @@ -241,6 +241,26 @@ Two details are load-bearing: against a driver built with another disagrees about the hardware shape with no diagnostic; rebuild the application with the same `CONFIGS` first. +### 6.1 Hardware profiles + +A hardware profile (`profiles/.json`) is a named machine layered between +the TOML defaults and the caller's flags. `PROFILE` — set by +`configure --profile=` or `blackbox.sh --profile=` — makes +`config.mk` fold the profile into `CONFIGS` once, so every consumer in the +table above sees one `CONFIGS`: + +```make +override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=... \ + --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +``` + +`--merge-configs` drops every profile entry for a knob the caller already sets +(recognising `X_DISABLE`/`X_ENABLE` and `X_`/`X=` as one knob), so +explicit flags win, then appends the caller's `CONFIGS` verbatim. The +generator validates a profile against the TOML: every key must be a public +knob, enum values must be declared, values are literals (expressions belong in +the TOML), and `sources` is required. + --- ## 7. The hardware/software boundary diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index c7f7e93962..118c5dbdcb 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -419,10 +419,17 @@ CP regfile is the only producer on every target. | `[33:26]` | 8 | log2 local_mem_size | `VX_CAPS_LOCAL_MEM_SIZE` | | `[36:34]` | 3 | log2 num_mem_banks | `VX_CAPS_NUM_MEM_BANKS` | | `[41:37]` | 5 | mem_bank_size, biased by 20 | `VX_CAPS_MEM_BANK_SIZE` | -| `[63:42]` | 22 | reserved | — | +| `[48:42]` | 7 | warps/core | `VX_CAPS_NUM_WARPS` | +| `[56:49]` | 8 | cluster_size | (only via `NUM_CORES`) | +| `[63:57]` | 7 | num_clusters | `VX_CAPS_NUM_CLUSTERS` | + +The exact counts in `[63:42]` let a machine that is not a power of two +(e.g. 48 warps, 84 cores) report its true shape; the decoder prefers them and +falls back to the log2 fields only when they read zero, as on devices built +before they existed. `VX_CAPS_NUM_CORES` is derived, not a field: -`num_clusters × cluster_size × socket_size`. `CACHE_LINE_SIZE`, +`num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not encoded here — `decode_caps` returns false and the backend resolves them. diff --git a/docs/designs/ray_tracing_architecture.md b/docs/designs/ray_tracing_architecture.md index cc100fc1eb..07b57b3318 100644 --- a/docs/designs/ray_tracing_architecture.md +++ b/docs/designs/ray_tracing_architecture.md @@ -176,7 +176,7 @@ state as base+count bursts, so each span's members must abut. | 17..22 | object-space ray `origin`/`dir` (staged on an AHS/IS yield) | RTU writes | | 23 / 24 / 25 | candidate `cb_type` / `sbt_idx` / `cb_handle` | RTU writes | | 26 | **STATUS** (`VX_RT_STS_*`) | RTU writes **last** | -| 27 / 29 | payload pointer / user `hitAttribute` | RTU writes (payload at arm; attr on commit) | +| 27 (+28 on RV64) / 29 | payload pointer (low / high word) / user `hitAttribute` | RTU writes (payload with every record; attr on commit) | **STATUS is deliberately last.** A parked WAIT completes on the address-match write to slot 26, so every other result word must already be in the window when diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 9913211a13..9da381bc39 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -118,7 +118,9 @@ atomics, threads, locks, or deferred-call plumbing — enforced by - `cache.{cpp,h}` — `Cache`: tags + MSHRs + replacement **and line data**; embeds the AMO unit. - `cache_cluster` — shared L2/L3 wrapper. -- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM (ramulator2); +- `memory.{cpp,h}` — `Memory`: bottom-of-hierarchy DRAM, one channel per + platform memory bank timed by `sim/common/dram_sim` (ramulator2, shared with + rtlsim/opaesim/xrtsim; see [Configuring the DRAM Model](../simulation.md#configuring-the-dram-model)); `attach_ram(RAM*)` owns the image; carries data in `MemRsp`; exposes `set_pre_send_hook` for SST mirroring. - `amo/amo_unit.{cpp,h}` — AMO RMW in the cache hierarchy (see diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 5a667d44a0..8b6a34a6e4 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -653,6 +653,10 @@ MMU moved out of the core there is no `sysmem_perf` slot to carry it, so the in-core view reads zero on rtlsim / FPGA (§13). Testing knobs: `VORTEX_RANDOMIZE_VA=0|1` and `VORTEX_VA_SEED=N` (§8.4); +`VORTEX_VM_MEGAPAGE_HEAP=1` maps the whole heap once with superpages at its +identity VA, as GPU drivers back allocations with 2 MB pages, so a buffer costs +no TLB entry per 4 KB page (per-buffer access flags are not enforced, and it +cannot be combined with `VORTEX_RANDOMIZE_VA`); `VX_CFG_VM_ADDR_MODE=BARE` in `CONFIGS` builds the VM hardware but runs untranslated. diff --git a/docs/install_vortex.md b/docs/install_vortex.md index d473aa34e3..d849384e75 100644 --- a/docs/install_vortex.md +++ b/docs/install_vortex.md @@ -40,8 +40,10 @@ sourcing is required and multiple Vortex trees can coexist on one machine without a global `~/.bashrc` clobber. - Useful configure options: `--xlen={32,64}` selects the ISA width and - `--prefix=` the install location. Two SimX build knobs are plain + Useful configure options: `--xlen={32,64}` selects the ISA width, + `--prefix=` the install location, and `--profile=` builds the + machine described by `profiles/.json` by default (see + [Hardware Profiles](simulation.md#hardware-profiles)). Two SimX build knobs are plain config defines, passed like any `VX_CFG_*` option through `CONFIGS`: `-DSIMX_MT=` runs the simulator on `T` host threads (default serial; cycle results are bit-identical for every thread count), and diff --git a/docs/simulation.md b/docs/simulation.md index 29c6d3a2de..8fbc55f94d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -138,6 +138,77 @@ PERF: instrs=363180, cycles=53108, IPC=6.838518 > to compare configurations relative to one another, not as fixed targets to > reproduce. +### Hardware Profiles + +A hardware profile is a named machine in `profiles/.json`: a set of +`VX_CFG_*` values, grouped into `core`, `chip` and `platform` sections, with a +`sources` entry saying where each value comes from (optionally also a +`reference` block of measurements from the real part, and `extends` to build +on another profile). Every flow that includes `config.mk` builds the same +machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. + +| Profile | Machine | +|---|---| +| `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | +| `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | +| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 1752 MT/s | +| `a100` | A100 (Ampere): 108 cores, 32 MB L2, HBM2 at the card's 1555 GB/s | +| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 at the card's 3.35 TB/s | +| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | + +Select one per build with `../configure --profile=`, or per run: + +```bash +$ ./ci/blackbox.sh --driver=simx --profile=rtx5080 --app=sgemm --args="-n256" +``` + +`CONFIGS` and blackbox's own `--clusters/--cores/--warps/--threads/--l2cache/--l3cache` +flags win over the profile: the profile only supplies the knobs they leave +unset, and everything else falls back to `VX_config.toml`. + +```bash +$ CONFIGS="-DVX_CFG_L2_SIZE=8388608" ./ci/blackbox.sh --driver=simx --profile=a100 --warps=32 --app=sgemm +``` + +The other profiles (20 to 132 cores) are SimX-scale machines for architecture +studies, not FPGA builds; `tinygpu` is the small one, sized for FPGA +bring-up and fast simulation. + +### Configuring the DRAM Model + +The simulators (simx, rtlsim, opaesim, xrtsim) all model the device memory +the same way: one independent DRAM channel (controller and device) per +platform memory bank, timed by +[Ramulator](https://github.com/CMU-SAFARI/ramulator2) with its JEDEC device +models. Each channel maps addresses the way GPU memory controllers do: four +bursts per row visit, then the next bank, with the bank permuted by the row so +strided streams spread over the banks. Refresh is not modelled: GPU +controllers hide it with per-bank refresh, for which Ramulator has no manager. A channel is described the way a data sheet describes it; these +`VX_config.toml` keys select it, overridable through `CONFIGS` or a profile, +and the RTL never reads them: + +| Key | Default | Meaning | +|---|---|---| +| `VX_CFG_PLATFORM_MEMORY_TYPE` | `HBM2` | JEDEC standard: `HBM2` (also HBM2E), `HBM3` or `GDDR6` (x16 channels) | +| `VX_CFG_PLATFORM_MEMORY_NUM_BANKS` | 2 | memory banks, and so DRAM channels (HBM: 128-bit channels) | +| `VX_CFG_PLATFORM_MEMORY_DATA_RATE` | 2 x `PLATFORM_CLOCK_RATE` | transfer rate per pin in MT/s; the default clocks the DRAM with the core | +| `VX_CFG_PLATFORM_CLOCK_RATE` | 400 | core clock in MHz; with the data rate it sets how many DRAM cycles a core cycle spans | +| `VX_CFG_PLATFORM_MEMORY_LATENCY` | 0 | fixed read latency added for the PHY, controller and interconnect, in core cycles | +| `VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE` | 0 | requests a channel holds in flight (0 = unbounded) | + +The model's peak bandwidth follows from the keys: channels x channel width x +data rate (HBM 16 B, GDDR6 2 B per transfer); `VX_CFG_PLATFORM_MEMORY_PEAK_BW` +separately states the platform's, which the runtime reports. The device's +array timings are held in nanoseconds, so a different data rate changes the +bandwidth but not the latency. The default machine clocks the DRAM with the +core: an idle read takes 14 cycles on an open row and 26 on a row conflict, +and the memory delivers up to one 64 B block per cycle. + +```bash +$ CONFIGS="-DVX_CFG_PLATFORM_MEMORY_TYPE=GDDR6 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_DATA_RATE=14000" \ + ./ci/blackbox.sh --driver=simx --app=vecadd +``` + ## Additional Quick Start Scenarios Running Vortex simulators with different configurations and drivers is supported. For example: diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index fa68f07398..86fc979410 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -43,8 +43,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - // Device MMU sideband from the top DCR surface. - input wire [`VX_CFG_XLEN-1:0] mmu_satp, // Device-level walker: the L2 TLB's miss bus, exported to the device. VX_tlb_bus_if.master dev_ptw_if, // Walker PTE fetches, entering as an L2 client when this cluster's L2 diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 762e1e0200..2b4b49df89 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -346,7 +346,6 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif `ifdef VX_CFG_VM_ENABLE - .mmu_satp (mmu_satp), .dev_ptw_if (per_cluster_dev_ptw_if[cluster_id]), .dev_ptw_mem_if (per_cluster_ptw_mem_if[cluster_id]), .mmu_flush_req (mmu_flush_req), diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index e9ba15f39e..61ad02a8e4 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -356,6 +356,9 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .axil_s (cp_axil), .axi_host (cp_axi_host), .axi_dev (cp_axi_dev), + // CCI-P has no AXI host port, so there are no drain counters to report. + .dbg_host_w_counts ('0), + .dbg_host_r_counts ('0), .gpu_if (cp_gpu_if), .irq (cp_interrupt) ); diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 23c33a60cd..63a857d330 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -101,8 +101,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // commit outputs (tied off when IS_LLC=0) output wire amo_hit_st1, // AMO commits locally at S1 output wire commit_busy, // commit in flight - output wire chain_stall, // pace same-line chained AMO + output wire chain_stall, // pace a request behind the compute stage output wire wb_pending, // writeback request live + output wire wbq_full, // no queue room for another chain's AMO output wire [WORD_WIDTH-1:0] rsp_data, // response word on amo_hit_st1 // Read forward: a request reading the AMO'd word while the result is still // queued/settling must observe those bytes (replays are admitted during @@ -152,6 +153,18 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end wire st0_ld_alloc_any = mshr_allocate_st0 && ~pipe_stall && ~amo_st0.amo_valid && ~rw_st0; wire st0_amo_alloc_any = mshr_allocate_st0 && ~pipe_stall && amo_st0.amo_valid; + // The allocation one stage on: at S1 it finalizes, and the MSHR probe + // only sees it the cycle after, so this covers the cycle in between. + reg st1_ld_alloc_r, st1_amo_alloc_r; + always @(posedge clk) begin + if (reset) begin + st1_ld_alloc_r <= 1'b0; + st1_amo_alloc_r <= 1'b0; + end else if (~pipe_stall) begin + st1_ld_alloc_r <= mshr_allocate_st0 && ~amo_st0.amo_valid && ~rw_st0; + st1_amo_alloc_r <= mshr_allocate_st0 && amo_st0.amo_valid; + end + end if (IS_LLC != 0) begin : g_commit // ---------------------------------------------------------------- @@ -184,8 +197,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // bounded by the core-port fan-in (<= 4 for this cache family), not the // full line. Sizing to that bound (min 2, cap 4) halves the coalesce/ // forward/enqueue fanout vs a full-line queue; the cap is a constant - // here because the engine has no NUM_REQS parameter, and the overflow - // assertion guards it if a wider core-port config ever exceeds 4. + // here because the engine has no NUM_REQS parameter. Replays bypass + // commit_busy, so fills are admitted against queue room (wbq_full). localparam WBQ_TARGETS = WORDS_PER_LINE * NUM_LANES; localparam WBQ_SIZE = (WBQ_TARGETS < 4) ? ((WBQ_TARGETS < 2) ? 2 : WBQ_TARGETS) : 4; localparam WBQ_CNTW = `CLOG2(WBQ_SIZE+1); @@ -228,9 +241,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire post_wb_valid = (post_wb_age != 2'd0); // Compute stage: S1 latches the aligned operands, the RMW ALU + the - // re-align shift run the next cycle, off the S1 critical path. AMO - // commits are serialized by commit_busy (the bank holds off core - // requests), so the stage holds at most one operation; the old operand + // re-align shift run the next cycle, off the S1 critical path. The + // stage holds at most one operation (commit_busy serializes core AMOs, + // chain_stall paces any AMO behind a busy stage); the old operand // is byte-forwarded from the writeback queue when a prior AMO to the same // bytes has not yet reached the array (see line_lane_st1). reg cmp_valid; @@ -750,7 +763,8 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // line AMOs back to back); those are paced instead by chain_stall. assign commit_busy = do_store_st0 || amo_inflight || do_store_st1 || cmp_valid || wb_pending_r; // Pace any same-line request sitting behind an in-flight compute by one - // cycle, so the result lands in wb_data_r and forwards cleanly. Gated on + // cycle, so the result lands in wb_data_r and forwards cleanly, and any + // AMO, which needs the compute stage free. Gated on // cmp_valid (an AMO is computing), so it never fires for baseline traffic. // The address match is registered from the look-ahead operands: both only // move on ~pipe_stall, so a held match stays valid across a stall. The @@ -765,7 +779,34 @@ module VX_cache_amo import VX_gpu_pkg::*; #( chain_match_r <= (cmp_addr_n == res_addr_n); end end - assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && chain_match_r; + assign chain_stall = cmp_valid && valid_st1 && is_creq_st1 && (chain_match_r || amo_st1.amo_valid); + + // AMOs that may reach the queue: at S0, between S0 and stC, and in the + // compute stage. A fill is admitted only while one more fits. The sum + // grows by at most one a cycle (one AMO enters S0), so the registered + // flag tests one entry early and stays exact a cycle later. + localparam AMO_CNTW = `CLOG2(PIPE_EX + 2); + wire st0_amo = amo_st0.amo_valid && valid_st0 && is_creq_st0; + wire st1_amo = amo_st1.amo_valid && valid_st1 && is_creq_st1; + wire amo_cnt_inc = st0_amo && ~pipe_stall; + wire amo_cnt_dec = st1_amo && ~pipe_stall; + reg [AMO_CNTW-1:0] amo_pipe_cnt; + reg wbq_full_r; + always @(posedge clk) begin + if (reset) begin + amo_pipe_cnt <= '0; + wbq_full_r <= 1'b0; + end else begin + if (amo_cnt_inc != amo_cnt_dec) begin + amo_pipe_cnt <= amo_cnt_inc ? (amo_pipe_cnt + AMO_CNTW'(1)) + : (amo_pipe_cnt - AMO_CNTW'(1)); + end + wbq_full_r <= (32'(wbq_count) + 32'(amo_pipe_cnt) + 32'(cmp_valid) + 32'(st0_amo) + 1) >= WBQ_SIZE; + end + end + assign wbq_full = wbq_full_r; + `RUNTIME_ASSERT (~(amo_cnt_dec && ~amo_cnt_inc && (amo_pipe_cnt == '0)), + ("%t: AMO pipe count underflow", $time)) // Invariants: a store-bearing AMO is only ever accepted into a free // compute stage (the queue absorbs different-line writebacks behind it), @@ -808,8 +849,9 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // covers it from S0 onward. Consumes only the registered probe (see // the header block) so the CAM stays out of core_req_ready. assign req_input_defer = core_req_valid - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); + `UNUSED_VAR (st1_ld_alloc_r) `UNUSED_VAR (amo_st0) // only amo_valid/amo_op are consumed at S0 `UNUSED_VAR (is_replay_st0) @@ -860,13 +902,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // Age-ordering guards, off the registered probe (see the header // block): an incoming AMO holds behind pending line-filling requests, - // an incoming load holds behind pending AMO passthroughs. The S0 - // alloc pulses catch the window between admit and allocate. + // an incoming load holds behind pending AMO passthroughs. The S0/S1 + // alloc terms catch the window between admit and the probe. wire amo_input_defer = core_req_valid && core_req_is_amo - && (st0_ld_alloc_any || persist_ld_r + && (st0_ld_alloc_any || st1_ld_alloc_r || persist_ld_r || (fresh_r ? mshr_any_ld : probe_ld_hit_r)); wire load_input_defer = core_req_valid && ~core_req_is_amo && ~core_req_rw - && (st0_amo_alloc_any || persist_amo_r + && (st0_amo_alloc_any || st1_amo_alloc_r || persist_amo_r || (fresh_r ? mshr_any_amo : probe_amo_hit_r)); assign req_input_defer = amo_input_defer || load_input_defer; @@ -875,6 +917,7 @@ module VX_cache_amo import VX_gpu_pkg::*; #( assign commit_busy = 1'b0; assign chain_stall = 1'b0; assign wb_pending = 1'b0; + assign wbq_full = 1'b0; assign rsp_data = '0; // no local commit window: loads are held at the input while a // forwarded AMO is pending on the line, so no read forward exists. diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index f62c57214f..28576fcd39 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -188,7 +188,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( amo_req_t replay_amo; // AMO engine interconnect (tied to 0 when the bank carries no AMO logic). - wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending; + wire amo_hit_st1, amo_commit_busy, amo_chain_stall, amo_wb_pending, amo_wbq_full; wire [`CS_WORD_WIDTH-1:0] amo_rsp_data; wire [WORD_SIZE-1:0] amo_rd_fwd_mask; wire [`CS_WORD_WIDTH-1:0] amo_rd_fwd_data; @@ -282,7 +282,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end - wire no_pending_req = ~pipe_inflight && mreq_queue_empty; + // Queued AMO results are still writes to land in the array. + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; VX_cache_flush #( .BANK_ID (BANK_ID), @@ -350,8 +351,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // A fill starts a replay chain, and a chain carries at most one AMO (no + // request enters behind a pending AMO), so a fill waits until the AMO + // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold; + wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -369,7 +373,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; @@ -997,6 +1001,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .commit_busy (amo_commit_busy), .chain_stall (amo_chain_stall), .wb_pending (amo_wb_pending), + .wbq_full (amo_wbq_full), .rsp_data (amo_rsp_data), .rd_fwd_mask (amo_rd_fwd_mask), .rd_fwd_data (amo_rd_fwd_data), @@ -1015,7 +1020,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .req_input_defer (req_input_defer) ); end else begin : g_no_amo - assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall} = '0; + assign {amo_hit_st1, amo_commit_busy, amo_wb_pending, amo_chain_stall, amo_wbq_full} = '0; assign {amo_rd_fwd_mask, amo_rd_fwd_data} = '0; assign {amo_rsp_data, amo_wb_addr, amo_wb_word_idx, amo_wb_byteen} = '0; assign {amo_wb_data, amo_wb_tag, amo_wb_idx, amo_wb_attr} = '0; diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index fdbdcaa787..22605fd148 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -373,6 +373,11 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + // At the LLC no request links behind a pending AMO: the bank defers + // every request to a line while an AMO to it waits here. + `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending + && valid_table[finalize_previd] && persisted_table[finalize_previd] && amo_table[finalize_previd]), + ("%t: *** %s request linked behind a pending AMO: id=%0d", $time, INSTANCE_ID, finalize_previd)) wire [MSHR_SIZE-1:0] probe_ld, probe_amo; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_probe_matches diff --git a/hw/rtl/cache/VX_cache_repl.sv b/hw/rtl/cache/VX_cache_repl.sv index 5aa92f7d5a..b639ac33ac 100644 --- a/hw/rtl/cache/VX_cache_repl.sv +++ b/hw/rtl/cache/VX_cache_repl.sv @@ -108,6 +108,7 @@ module VX_cache_repl #( if (REPL_POLICY == `CS_REPL_PLRU) begin : g_plru // Pseudo Least Recently Used replacement policy `UNUSED_VAR (repl_line) // read uses repl_line_n (look-ahead) + `UNUSED_VAR (repl_valid) // state advances on lookup hits only localparam LRU_WIDTH = `UP(NUM_WAYS-1); wire [LRU_WIDTH-1:0] plru_rdata; diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index 51efaa9762..f09c17bb7e 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -338,7 +338,7 @@ import VX_fpu_pkg::*; end end `else - `UNUSED_VAR (write_sel[WR_FFLAGS +: 3]) + `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_slice.sv index dba90d2856..affb932298 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_slice.sv @@ -42,9 +42,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( // window: the LMEM base is asserted aligned to the capacity, so the prefix // compare is exact and needs no end address -- which is not representable // when the window ends at the top of the address space. +`ifdef VX_CFG_LMEM_ENABLE localparam LMEM_BLK_LSB = `VX_CFG_LMEM_LOG_SIZE - MEM_ASHIFT; localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) +`endif `UNUSED_PARAM (CORE_ID) `UNUSED_SPARAM (INSTANCE_ID) diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index db425cbb2c..806a996da9 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -128,8 +128,13 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + ("GPU_DEV_CAPS exact-count fields overflow")) + wire [63:0] gpu_dev_caps = { - 22'b0, + 7'(`VX_CFG_NUM_CLUSTERS), + 8'(GPU_CLUSTER_SIZE), + 7'(`VX_CFG_NUM_WARPS), 5'(GPU_BANK_ADDR_W - 20), 3'($clog2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS)), 8'(`VX_CFG_LMEM_ENABLED ? `VX_CFG_LMEM_LOG_SIZE : 0), diff --git a/hw/rtl/fpu/VX_fma_unit_rtl.sv b/hw/rtl/fpu/VX_fma_unit_rtl.sv index cddf2b4889..02b986d826 100644 --- a/hw/rtl/fpu/VX_fma_unit_rtl.sv +++ b/hw/rtl/fpu/VX_fma_unit_rtl.sv @@ -115,9 +115,9 @@ module VX_fma_unit_rtl import VX_gpu_pkg::*, VX_fpu_pkg::*; #( // DAZ: flush a subnormal operand to signed zero when subnormals are disabled. localparam DAZ = (SUBNORM_ENABLE == 0); - function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] v); - daz_f = (DAZ && (v[FLOAT_BITS-2:MAN_BITS] == '0) && (|v[MAN_BITS-1:0])) - ? {v[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : v; + function automatic [FLOAT_BITS-1:0] daz_f(input [FLOAT_BITS-1:0] val); + daz_f = (DAZ && (val[FLOAT_BITS-2:MAN_BITS] == '0) && (|val[MAN_BITS-1:0])) + ? {val[FLOAT_BITS-1], {(FLOAT_BITS-1){1'b0}}} : val; endfunction wire [FLOAT_BITS-1:0] op_a = daz_f(dataa); diff --git a/hw/rtl/libs/VX_find_first.sv b/hw/rtl/libs/VX_find_first.sv index b7fc3e2596..92ad0f2d20 100644 --- a/hw/rtl/libs/VX_find_first.sv +++ b/hw/rtl/libs/VX_find_first.sv @@ -28,32 +28,30 @@ module VX_find_first #( localparam TL = (1 << LOGN) - 1; localparam TN = (1 << (LOGN+1)) - 1; - wire s_n [TN]; - wire [DATAW-1:0] d_n [TN]; - - for (genvar i = 0; i < N; ++i) begin : g_fill - assign s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; - assign d_n[TL+i] = REVERSE ? data_in[N-1-i] : data_in[i]; - end - - if (TL < (TN-N)) begin : g_padding - for (genvar i = TL+N; i < TN; ++i) begin : g_i - assign s_n[i] = 0; - assign d_n[i] = '0; + reg valid_r; + reg [DATAW-1:0] data_r; + + // Binary tree: node k reduces children 2k+1 (first) and 2k+2; leaves start at TL. + // The tree is local to one block so no node variable feeds itself. + always @(*) begin : g_scan + reg [TN-1:0] s_n; + reg [TN*DATAW-1:0] d_n; + s_n = '0; + d_n = '0; + for (integer i = 0; i < N; ++i) begin + s_n[TL+i] = REVERSE ? valid_in[N-1-i] : valid_in[i]; + d_n[(TL+i)*DATAW +: DATAW] = REVERSE ? data_in[N-1-i] : data_in[i]; end - end - - for (genvar j = 0; j < LOGN; ++j) begin : g_scan - localparam I = 1 << j; - for (genvar i = 0; i < I; ++i) begin : g_i - localparam K = I+i-1; - assign s_n[K] = s_n[2*K+2] | s_n[2*K+1]; - assign d_n[K] = s_n[2*K+1] ? d_n[2*K+1] : d_n[2*K+2]; + for (integer k = TL-1; k >= 0; --k) begin + s_n[k] = s_n[2*k+2] | s_n[2*k+1]; + d_n[k*DATAW +: DATAW] = s_n[2*k+1] ? d_n[(2*k+1)*DATAW +: DATAW] : d_n[(2*k+2)*DATAW +: DATAW]; end + valid_r = s_n[0]; + data_r = d_n[0 +: DATAW]; end - assign valid_out = s_n[0]; - assign data_out = d_n[0]; + assign valid_out = valid_r; + assign data_out = data_r; endmodule `TRACING_ON diff --git a/hw/rtl/libs/VX_onehot_encoder.sv b/hw/rtl/libs/VX_onehot_encoder.sv index 273a9cfe26..2bbf58d377 100644 --- a/hw/rtl/libs/VX_onehot_encoder.sv +++ b/hw/rtl/libs/VX_onehot_encoder.sv @@ -39,31 +39,38 @@ module VX_onehot_encoder #( end else if (MODEL == 1) begin : g_model1 localparam M = 1 << LN; - wire [M-1:0] addr [LN]; - wire [M-1:0] v [LN+1]; - - // base case, also handle padding for non-power of two inputs - assign v[0] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); - - for (genvar lvl = 1; lvl < (LN+1); ++lvl) begin : g_scan_l - localparam SN = 1 << (LN - lvl); - localparam SI = M / SN; - for (genvar s = 0; s < SN; ++s) begin : g_scan_s - wire [1:0] vs = {v[lvl-1][s*SI+(SI>>1)], v[lvl-1][s*SI]}; - assign v[lvl][s*SI] = (| vs); - if (lvl == 1) begin : g_lvl_1 - assign addr[lvl-1][s*SI +: lvl] = vs[!REVERSE]; - end else begin : g_lvl_n - assign addr[lvl-1][s*SI +: lvl] = { - vs[!REVERSE], - addr[lvl-2][s*SI +: lvl-1] | addr[lvl-2][s*SI+(SI>>1) +: lvl-1] - }; + + reg [LN-1:0] addr_r; + reg valid_r; + + // Level lvl merges pairs of level lvl-1 groups of size 2^(lvl-1); each + // level has its own slice of v/addr, local to one block, so no + // variable feeds itself. + always @(*) begin : g_scan + reg [(LN+1)*M-1:0] v; + reg [LN*M-1:0] addr; + reg [1:0] vs; + v = '0; + addr = '0; + // base case, also handle padding for non-power of two inputs + v[0 +: M] = REVERSE ? (M'(data_in) << (M - N)) : M'(data_in); + for (integer lvl = 1; lvl <= LN; ++lvl) begin + for (integer s = 0; s < (1 << (LN - lvl)); ++s) begin + vs = {v[(lvl-1)*M + s*(1 << lvl) + (1 << (lvl-1))], v[(lvl-1)*M + s*(1 << lvl)]}; + v[lvl*M + s*(1 << lvl)] = (| vs); + addr[(lvl-1)*M + s*(1 << lvl) + (lvl-1)] = vs[!REVERSE]; + for (integer b = 0; b < lvl-1; ++b) begin + addr[(lvl-1)*M + s*(1 << lvl) + b] = addr[(lvl-2)*M + s*(1 << lvl) + b] + | addr[(lvl-2)*M + s*(1 << lvl) + (1 << (lvl-1)) + b]; + end end end + addr_r = addr[(LN-1)*M +: LN]; + valid_r = v[LN*M]; end - assign data_out = addr[LN-1][LN-1:0]; - assign valid_out = v[LN][0]; + assign data_out = addr_r; + assign valid_out = valid_r; end else if (MODEL == 2 && REVERSE == 0) begin : g_model2 diff --git a/hw/rtl/libs/VX_pipe_buffer.sv b/hw/rtl/libs/VX_pipe_buffer.sv index 0847c0bbe9..e72e8d2b35 100644 --- a/hw/rtl/libs/VX_pipe_buffer.sv +++ b/hw/rtl/libs/VX_pipe_buffer.sv @@ -40,7 +40,7 @@ module VX_pipe_buffer #( assign data_out = data_in; end else begin : g_register wire [DEPTH:0] valid; - wire ready [DEPTH+1]; + wire [DEPTH-1:0] ready; wire [DEPTH:0][DATAW-1:0] data; assign valid[0] = valid_in; @@ -48,7 +48,8 @@ module VX_pipe_buffer #( assign ready_in = ready[0]; for (genvar i = 0; i < DEPTH; ++i) begin : g_pipe_regs - assign ready[i] = (ready[i+1] || ~valid[i+1]); + // Stage i advances when any later stage has a hole or the output drains. + assign ready[i] = ready_out || ~(& valid[DEPTH:i+1]); VX_pipe_register #( .DATAW (1 + DATAW), .RESETW (1 + RESETW) @@ -63,7 +64,6 @@ module VX_pipe_buffer #( assign valid_out = valid[DEPTH]; assign data_out = data[DEPTH]; - assign ready[DEPTH] = ready_out; end endmodule diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 59ae4e853d..2cb0529650 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -141,7 +141,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX assign pix_cand_n[p] = zbits[31] ? 24'd0 : (|zbits[30:24]) ? 24'(OM_DEPTH_MASK) : zbits[23:0]; - wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? dcrs.zbuf_pitch : 32'd0); + wire [31:0] y_pitch = quad_ypitch[q] + (i[1] ? 32'(dcrs.zbuf_pitch) : 32'd0); wire [W_ADDR_BITS-1:0] baddr = {dcrs.zbuf_addr, 4'b0} + W_ADDR_BITS'(px); assign pix_waddr_n[p] = baddr + W_ADDR_BITS'(y_pitch[31:2]); `UNUSED_VAR (y_pitch) diff --git a/hw/rtl/rtu/VX_rtu_bus_arb.sv b/hw/rtl/rtu/VX_rtu_bus_arb.sv index f619044b3c..ec01860259 100644 --- a/hw/rtl/rtu/VX_rtu_bus_arb.sv +++ b/hw/rtl/rtu/VX_rtu_bus_arb.sv @@ -68,7 +68,7 @@ module VX_rtu_bus_arb import VX_gpu_pkg::*, VX_rtu_pkg::*; #( localparam SRC_IN_W = 1; localparam ARM_DATAW = NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; // the win payload carries the sel-stripped tag on the core side localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_bus_if.sv b/hw/rtl/rtu/VX_rtu_bus_if.sv index 23b017e826..2f259fa299 100644 --- a/hw/rtl/rtu/VX_rtu_bus_if.sv +++ b/hw/rtl/rtu/VX_rtu_bus_if.sv @@ -69,7 +69,7 @@ interface VX_rtu_bus_if import VX_gpu_pkg::*, VX_rtu_pkg::*; #( logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] scene_base; // device address of the scene logic [15:0] flags; // VX_RT_FLAG_* logic [15:0] cull_mask; - logic [31:0] payload_ptr; // the RTU stages it for the callbacks + logic [`VX_CFG_XLEN-1:0] payload_ptr; // the RTU stages it for the callbacks logic [TAG_WIDTH-1:0] tag; } arm_data_t; diff --git a/hw/rtl/rtu/VX_rtu_bus_slice.sv b/hw/rtl/rtu/VX_rtu_bus_slice.sv index d2601157a0..a85e27711c 100644 --- a/hw/rtl/rtu/VX_rtu_bus_slice.sv +++ b/hw/rtl/rtu/VX_rtu_bus_slice.sv @@ -54,7 +54,7 @@ module VX_rtu_bus_slice import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // parameter cannot read $bits of a hierarchical interface member, so size // the elastic buffers from the field formula instead. localparam ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH; + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH; localparam REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS; localparam WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/hw/rtl/rtu/VX_rtu_core.sv b/hw/rtl/rtu/VX_rtu_core.sv index d2dedd1238..3f018a13e0 100644 --- a/hw/rtl/rtu/VX_rtu_core.sv +++ b/hw/rtl/rtu/VX_rtu_core.sv @@ -156,11 +156,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( reg [NUM_SLOTS-1:0][2:0] tstate; reg [NUM_SLOTS-1:0][NUM_LANES-1:0] req_mask; - reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; - reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from - reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write - reg [NUM_SLOTS-1:0][31:0] req_payload; // warp-uniform; staged for the callbacks - reg [NUM_SLOTS-1:0] is_cand; + reg [NUM_SLOTS-1:0][TAG_WIDTH-1:0] req_tag; + reg [NUM_SLOTS-1:0][STG_IDX_W-1:0] req_stg; // the staging entry it launched from + reg [NUM_SLOTS-1:0][NW_WIDTH-1:0] req_wid; // its warp, for the window write + reg [NUM_SLOTS-1:0][`VX_CFG_XLEN-1:0] req_payload; // warp-uniform; staged for the callbacks + reg [NUM_SLOTS-1:0] is_cand; reg [NUM_SLOTS-1:0][NUM_LANES-1:0][RTU_CB_ACTION_BITS-1:0] cont_action; // ── the scheduler ───────────────────────────────────────────────── @@ -316,7 +316,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( ); // warp-uniform scalars of a staged trace (read once, at the claim) - localparam SCLW = NUM_LANES + TAG_WIDTH + 32 + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; + localparam SCLW = NUM_LANES + TAG_WIDTH + `VX_CFG_XLEN + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16; wire [SCLW-1:0] stg_scl_rdata; wire [SLOT_W-1:0] fill_slot_pick; wire [STG_IDX_W-1:0] fill_stg_pick; @@ -342,11 +342,11 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .raddr (fill_stg_pick), .rdata (stg_scl_rdata) ); - wire [NUM_LANES-1:0] scl_mask; - wire [TAG_WIDTH-1:0] scl_tag; - wire [31:0] scl_payload; + wire [NUM_LANES-1:0] scl_mask; + wire [TAG_WIDTH-1:0] scl_tag; + wire [`VX_CFG_XLEN-1:0] scl_payload; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] scl_scene; - wire [15:0] scl_flags, scl_cull; + wire [15:0] scl_flags, scl_cull; assign {scl_mask, scl_tag, scl_payload, scl_scene, scl_flags, scl_cull} = stg_scl_rdata; // ── the fill engine ─────────────────────────────────────────────── @@ -491,7 +491,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // 7..12 object ray (candidate only) — the staging RAM's beat rows // 13,14 cb_type (flag flops), sbt (window row) // 15 cb_handle (0) - // then payload (descriptor), hitAttribute (window row), status LAST + // then payload (descriptor, XLEN/32 words), hitAttribute (window row), status LAST localparam [2:0] WB_IDLE = 3'd0, WB_RD1 = 3'd1, WB_CAP1 = 3'd2, @@ -505,9 +505,10 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [RTU_IDX_BITS-1:0] n_attrs = is_cand[ws] ? RTU_IDX_BITS'(RTU_RES_CAND) : RTU_IDX_BITS'(RTU_RES_HIT); - wire wr_payload = (wr_idx == n_attrs); - wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); - wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(2))); + wire wr_pl_hi = (RTU_PAYLOAD_WORDS > 1) && (wr_idx == (n_attrs + RTU_IDX_BITS'(1))); + wire wr_payload = (wr_idx == n_attrs) || wr_pl_hi; + wire wr_attr = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS))); + wire wr_status = (wr_idx == (n_attrs + RTU_IDX_BITS'(RTU_PAYLOAD_WORDS + 1))); wire wr_hitspan = (wr_idx < RTU_IDX_BITS'(RTU_RES_HIT)); wire wr_objray = is_cand[ws] && !wr_hitspan && (wr_idx < RTU_IDX_BITS'(13)); wire wr_cbtype = is_cand[ws] && (wr_idx == RTU_IDX_BITS'(13)); @@ -570,7 +571,8 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wr_status) begin win_word[i] = status_word[i]; end else if (wr_payload) begin - win_word[i] = req_payload[ws]; + win_word[i] = wr_pl_hi ? req_payload[ws][`VX_CFG_XLEN-1 -: 32] + : req_payload[ws][31:0]; end else if (wr_attr) begin win_word[i] = sch_attrv[wctx[i]] ? wb_d1[i*32 +: 32] : 32'd0; end else if (wr_hitspan) begin @@ -608,7 +610,7 @@ module VX_rtu_core import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.win_data.mask = (is_cand[ws] && !wr_uniform) ? wr_cb_mask : req_mask[ws]; assign rtu_bus_w.win_data.slot = wr_status ? RTU_SLOT_BITS'(RTU_STATUS_SLOT) - : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + : wr_payload ? RTU_SLOT_BITS'(RTU_PAYLOAD_SLOT) + RTU_SLOT_BITS'(wr_pl_hi) : wr_attr ? RTU_SLOT_BITS'(RTU_ATTR_SLOT) : RTU_SLOT_BITS'(RTU_RES_BASE) + RTU_SLOT_BITS'(wr_idx); diff --git a/hw/rtl/rtu/VX_rtu_pkg.sv b/hw/rtl/rtu/VX_rtu_pkg.sv index 4a061f53b5..2f31dfe102 100644 --- a/hw/rtl/rtu/VX_rtu_pkg.sv +++ b/hw/rtl/rtu/VX_rtu_pkg.sv @@ -263,21 +263,22 @@ package VX_rtu_pkg; // // hit attrs [RES_BASE, +RES_HIT) written by every response // candidate [RES_BASE, +RES_CAND) + object ray, cb_type, sbt, handle - // payload PAYLOAD_SLOT written once, at arm (warp-uniform) + // payload PAYLOAD_SLOT(+1) XLEN/32 words, written per record (warp-uniform) // hit attr ATTR_SLOT the IS shader's hitAttribute, on commit // status STATUS_SLOT written LAST // // Status is last by design: writing it is what completes the warp's parked // WAIT, so every slot the warp may then read must already be in place. // ───────────────────────────────────────────────────────────────── - localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max - localparam RTU_RES_BASE = `VX_RT_HIT_T; - localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom - localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle - localparam RTU_STATUS_SLOT = `VX_RT_STATUS; - localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; - localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; - localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); + localparam RTU_RAY_BEATS = 8; // origin[3], dir[3], t_min, t_max + localparam RTU_RES_BASE = `VX_RT_HIT_T; + localparam RTU_RES_HIT = 7; // hit_t, hit_u, hit_v, prim, instance, geometry, custom + localparam RTU_RES_CAND = 16; // + object ray (6), cb_type, cb_sbt_idx, cb_handle + localparam RTU_STATUS_SLOT = `VX_RT_STATUS; + localparam RTU_PAYLOAD_SLOT = `VX_RT_PAYLOAD_PTR_LO; + localparam RTU_PAYLOAD_WORDS = `VX_CFG_XLEN / 32; + localparam RTU_ATTR_SLOT = `VX_RT_HIT_ATTR_0; + localparam RTU_IDX_BITS = `CLOG2(RTU_RES_CAND + 1); // ───────────────────────────────────────────────────────────────── // Window store — the per-slot result record, one NUM_LANES*32 row per word, diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index 8477773ae5..46f1676251 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -119,8 +119,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( localparam LINES = FLAT ? RTU_FLAT_LINES : RTU_NODE_LINES; localparam LB = `CLOG2(LINES + 1); localparam BUF_BITS = LINES * LINE_BITS; - localparam IDXW = `CLOG2(`UP(RTU_BVH_WIDTH)); localparam NODE_W = `UP(RTU_BVH_WIDTH); + localparam IDXW = `LOG2UP(NODE_W); localparam ADDRW = `VX_CFG_MEM_ADDR_WIDTH; `STATIC_ASSERT(((NUM_LANES * NUM_SLOTS) == NUM_CTX), @@ -461,8 +461,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire recip_obj; wire recip_valid_out, recip_last_out; wire [CTX_TAG_W-1:0] recip_tag_out; - wire [1:0] recip_axis_out; - wire [31:0] recip_result; wire [95:0] recip_rdata; // ── short stack (BVH only) ──────────────────────────────────────── @@ -792,6 +790,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign ins_shift[oc] = ~ins_le[oc-1]; end end else begin : g_no_ins_slots + `UNUSED_VAR ({coll_ordt_sel, coll_ordcnt_sel}) assign {ins_le, ins_here, ins_shift} = '0; end wire coll_pushable = box_hit && (box_childoff != 32'd0); @@ -847,6 +846,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire [1:0] recip_axis; wire [31:0] recip_din = recip_obj ? word_q.obj_d[recip_axis] : ray_q.dir[recip_axis]; if (!FLAT) begin : g_recip_pe + wire [1:0] recip_axis_out; + wire [31:0] recip_result; VX_rtu_recip #( .LATENCY (RTU_FDIV_LAT), .DSP_SEED (`VX_CFG_RTU_RECIP_DSP_SEED) @@ -890,8 +891,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign recip_valid_out = 1'b0; assign recip_last_out = 1'b0; assign recip_tag_out = '0; - assign recip_axis_out = '0; - assign recip_result = '0; assign recip_rdata = '0; `UNUSED_VAR ({recip_issue, recip_obj, recip_din, recip_q}) end @@ -1292,7 +1291,11 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [LVLW-1:0] path_wlvl_r; reg [PATHW-1:0] path_wdata_r; - wire mem_fire = x_valid && mem_issue && mem_req_ready; + // The issue block below reads the grant only where it has just set + // mem_issue, so it uses mem_grant: reading mem_fire would feed its own + // output back into itself (a combinational loop to Verilator). + wire mem_grant = x_valid && mem_req_ready; + wire mem_fire = mem_grant && mem_issue; wire [RTU_CHILD_BITS-1:0] last_child = node.n_children - RTU_CHILD_BITS'(1); @@ -1355,7 +1358,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_HDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_HDR_WAIT; end else begin wake_self = 1'b1; @@ -1400,7 +1403,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSP0; end else begin wake_self = 1'b1; @@ -1435,7 +1438,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_RSPN; end else begin wake_self = 1'b1; @@ -1598,7 +1601,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_LTRI_REQ0: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = tri_rec_lines; word_n.cstate = CS_LTRI_RSP0; end else begin @@ -1619,7 +1622,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_LTRI_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_LTRI_RSPN; end else begin wake_self = 1'b1; @@ -1758,7 +1761,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_INST_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.f_total = inst_lines; word_n.cstate = CS_INST_RSP0; end else begin @@ -1777,7 +1780,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_INST_REQN: begin mem_issue = 1'b1; mem_fslot = word_x.f_idx; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_INST_RSPN; end else begin wake_self = 1'b1; @@ -1862,7 +1865,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end CS_BHDR_REQ: begin mem_issue = 1'b1; - if (mem_fire) begin + if (mem_grant) begin word_n.cstate = CS_BHDR_WAIT; end else begin wake_self = 1'b1; diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index 07bc926c74..cfeb22a315 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -204,7 +204,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( assign rtu_bus_w.arm_data.wid = wid; assign rtu_bus_w.arm_data.mask = execute_if.data.header.tmask; assign rtu_bus_w.arm_data.scene_base = execute_if.data.rs1_data[CFG_L1][`VX_CFG_MEM_ADDR_WIDTH-1:0]; - assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2][31:0]; + assign rtu_bus_w.arm_data.payload_ptr = execute_if.data.rs1_data[CFG_L2]; assign rtu_bus_w.arm_data.flags = execute_if.data.rs1_data[CFG_L3][15:0]; assign rtu_bus_w.arm_data.cull_mask = execute_if.data.rs1_data[CFG_L3][31:16]; assign rtu_bus_w.arm_data.tag = ($bits(rtu_bus_w.arm_data.tag))'(execute_if.data.header.uuid); @@ -391,8 +391,21 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // write back zero. sfu_result_t rsp_data_out; assign rsp_data_out.header = s1_header; +`ifdef VX_CFG_XLEN_64 +`ifdef VX_CFG_EXT_F_ENABLE + // A single-precision slot read into a 64-bit FP register must be NaN-boxed. + wire rsp_is_float = (get_reg_type(s1_header.rd) == REG_TYPE_F); +`else + wire rsp_is_float = 1'b0; +`endif +`endif for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data - assign rsp_data_out.data[i] = `VX_CFG_XLEN'(s1_from_ram ? core_rdata[i] : 32'd0); + wire [31:0] rsp_word = s1_from_ram ? core_rdata[i] : 32'd0; + `ifdef VX_CFG_XLEN_64 + assign rsp_data_out.data[i] = rsp_is_float ? {32'hffffffff, rsp_word} : 64'(rsp_word); + `else + assign rsp_data_out.data[i] = rsp_word; + `endif end assign result_if.valid = s1_valid; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv index dec2159349..12953c50f3 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv @@ -55,6 +55,10 @@ module VX_tcu_tfr_shared_mul import VX_tcu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `ifndef VX_CFG_TCU_MX_ENABLE `UNUSED_PARAM (SF) +`else + if (!`VX_CFG_TCU_FP8_ENABLED && !`VX_CFG_TCU_FP4_ENABLED) begin : g_sf_unused + `UNUSED_VAR ({sf_a, sf_b}) + end `endif `UNUSED_VAR ({clk, req_id, valid_in}) diff --git a/hw/rtl/tex/VX_tex_bus_arb.sv b/hw/rtl/tex/VX_tex_bus_arb.sv index 319570f9c6..5041b81a11 100644 --- a/hw/rtl/tex/VX_tex_bus_arb.sv +++ b/hw/rtl/tex/VX_tex_bus_arb.sv @@ -32,6 +32,8 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( ); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = TAG_WIDTH + NUM_LANES * (1 + 2 * 32 + TEX_LODF_BITS) + TEX_STAGE_BITS; localparam RSP_DATAW = TAG_WIDTH + NUM_LANES * 32; @@ -43,7 +45,7 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in @@ -72,15 +74,20 @@ module VX_tex_bus_arb import VX_gpu_pkg::*, VX_tex_pkg::*; #( for (genvar i = 0; i < NUM_OUTPUTS; ++i) begin : g_bus_out_if wire [TAG_WIDTH-1:0] req_tag_out; - VX_bits_insert #( - .N (TAG_WIDTH), - .S (LOG_NUM_REQS), - .POS (TAG_SEL_IDX) - ) bits_insert ( - .data_in (req_tag_out), - .ins_in (req_sel_out[i]), - .data_out (bus_out_if[i].req_data.tag) - ); + if (NUM_INPUTS > NUM_OUTPUTS) begin : g_req_tag_sel_out + VX_bits_insert #( + .N (TAG_WIDTH), + .S (LOG_NUM_REQS), + .POS (TAG_SEL_IDX) + ) bits_insert ( + .data_in (req_tag_out), + .ins_in (req_sel_out[i]), + .data_out (bus_out_if[i].req_data.tag) + ); + end else begin : g_req_tag_out + `UNUSED_VAR (req_sel_out) + assign bus_out_if[i].req_data.tag = req_tag_out; + end assign bus_out_if[i].req_valid = req_valid_out[i]; assign {req_tag_out, bus_out_if[i].req_data.mask, bus_out_if[i].req_data.coords, bus_out_if[i].req_data.lod, bus_out_if[i].req_data.stage} = req_data_out[i]; assign req_ready_out[i] = bus_out_if[i].req_ready; diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 06b4ea7f48..9166421400 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -67,9 +67,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam FIELDS_W = F_RW_LO + 1; localparam PAYLOAD_W = LANE_W + FIELDS_W; - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction function automatic tlb_access_e access_of (input logic rw); access_of = (EXEC_SIDE != 0) ? TLB_ACC_EX : (rw ? TLB_ACC_WR : TLB_ACC_RD); @@ -255,7 +252,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( if (ADDR_WIDTH > PAGE_OFFSET_W + TLB_VPN_WIDTH) begin : g_replay_va_hi `UNUSED_VAR (replay_raddr[ADDR_WIDTH-1 : PAGE_OFFSET_W + TLB_VPN_WIDTH]) end - wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~vpn_mask(replay_level); + wire [TLB_VPN_WIDTH-1:0] replay_low = replay_vpn & ~tlb_vpn_mask(replay_level); wire [TLB_PPN_WIDTH-1:0] replay_ppn_sp = replay_ppn | TLB_PPN_WIDTH'(replay_low); wire [ADDR_WIDTH-1:0] replay_taddr = {replay_ppn_sp, replay_raddr[PAGE_OFFSET_W-1:0]}; diff --git a/hw/rtl/vm/VX_tlb_cam.sv b/hw/rtl/vm/VX_tlb_cam.sv index ec63559c33..950d974626 100644 --- a/hw/rtl/vm/VX_tlb_cam.sv +++ b/hw/rtl/vm/VX_tlb_cam.sv @@ -56,11 +56,6 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( reg [TLB_SIZE-1:0] valid_r; reg [TLB_SIZE-1:0] mru_r; - // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). - function automatic [TLB_VPN_WIDTH-1:0] vpn_mask (input [TLB_LEVEL_WIDTH-1:0] level); - vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); - endfunction - // --------------------------------------------------------------------- // Per-lane lookup // --------------------------------------------------------------------- @@ -77,7 +72,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lookup wire [TLB_SIZE-1:0] hit_vec; for (genvar i = 0; i < TLB_SIZE; ++i) begin : g_match - wire [TLB_VPN_WIDTH-1:0] mask = vpn_mask(entries_r[i].level); + wire [TLB_VPN_WIDTH-1:0] mask = tlb_vpn_mask(entries_r[i].level); assign hit_vec[i] = valid_r[i] && ((entries_r[i].vpn & mask) == (lookup_vpn[l] & mask)); end @@ -115,7 +110,7 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[0 +: TLB_LEVEL_WIDTH]; - wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~vpn_mask(sel_level); + wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~tlb_vpn_mask(sel_level); // Splice the superpage's intra-page index from the VPN into the PPN. assign lookup_ppn[l] = sel_ppn | TLB_PPN_WIDTH'(low); assign lookup_flags[l] = sel_flags; diff --git a/hw/rtl/vm/VX_tlb_pkg.sv b/hw/rtl/vm/VX_tlb_pkg.sv index 8392a7ffbe..bc9aa17da4 100644 --- a/hw/rtl/vm/VX_tlb_pkg.sv +++ b/hw/rtl/vm/VX_tlb_pkg.sv @@ -72,6 +72,11 @@ package VX_tlb_pkg; && ( want_w || want_x || flags[TLB_FLAG_R]); endfunction + // Mask that drops the low level*TLB_LEVEL_BITS VPN bits (superpages). + function automatic logic [TLB_VPN_WIDTH-1:0] tlb_vpn_mask (input logic [TLB_LEVEL_WIDTH-1:0] level); + tlb_vpn_mask = {TLB_VPN_WIDTH{1'b1}} << (level * TLB_LEVEL_BITS); + endfunction + endpackage `endif // VX_TLB_PKG_VH diff --git a/hw/unittest/common.mk b/hw/unittest/common.mk index 3729fadce2..25ca8e25ee 100644 --- a/hw/unittest/common.mk +++ b/hw/unittest/common.mk @@ -3,8 +3,8 @@ DESTDIR ?= . CONFIGS += PARAMS += -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) @@ -67,8 +67,10 @@ endif all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' -MMD -o ../$@ run: $(DESTDIR)/$(PROJECT) $(DESTDIR)/$(PROJECT) $(OPTS) diff --git a/hw/unittest/dxa_core/main.cpp b/hw/unittest/dxa_core/main.cpp index 91b74ff863..6f796cd0aa 100644 --- a/hw/unittest/dxa_core/main.cpp +++ b/hw/unittest/dxa_core/main.cpp @@ -19,6 +19,7 @@ #include #include #include "VVX_dxa_core_top.h" +#include "VX_config.h" #include "verilated.h" #ifdef VCD_OUTPUT #include "verilated_vcd_c.h" @@ -93,9 +94,10 @@ int main(int argc, char** argv) { dut->dcr_req_valid = 0; dut->dcr_req_rw = 0; dut->dxa_req_valid = 0; - dut->gmem_req_ready = ~0ULL; // all ready + auto all_ready = [](uint32_t ports) { return (ports >= 32) ? ~0u : ((1u << ports) - 1); }; + dut->gmem_req_ready = all_ready(VX_CFG_NUM_DXA_CORES); dut->gmem_rsp_valid = 0; - dut->lmem_req_ready = ~0ULL; // all ready + dut->lmem_req_ready = all_ready(VX_CFG_SOCKET_SIZE); reset_dut(); diff --git a/hw/unittest/fma_unit/main.cpp b/hw/unittest/fma_unit/main.cpp index 2ff05c8a90..7f0c07b672 100644 --- a/hw/unittest/fma_unit/main.cpp +++ b/hw/unittest/fma_unit/main.cpp @@ -74,6 +74,9 @@ void encode(int op, uint32_t a, uint32_t b, uint32_t c, case FMSUB: op_type = OP_MADD; fmt = 2; break; // a*b - c case FNMADD: op_type = OP_NMADD; fmt = 0; break; // -(a*b) - c case FNMSUB: op_type = OP_NMADD; fmt = 2; break; // -(a*b) + c + default: + std::fprintf(stderr, "encode: unknown op %d\n", op); + std::abort(); } } diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 53386bfd77..51aed0b6d5 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -99,7 +99,7 @@ module VX_gfx_top import VX_gpu_pkg::* input wire [NC-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] rtu_arm_scene_base, input wire [NC-1:0][15:0] rtu_arm_flags, input wire [NC-1:0][15:0] rtu_arm_cull_mask, - input wire [NC-1:0][31:0] rtu_arm_payload_ptr, + input wire [NC-1:0][`VX_CFG_XLEN-1:0] rtu_arm_payload_ptr, input wire [NC-1:0][RTU_REQ_TAG_WIDTH-1:0] rtu_arm_tag, output wire [NC-1:0] rtu_arm_ready, diff --git a/hw/unittest/rtu_core/VX_rtu_core_top.sv b/hw/unittest/rtu_core/VX_rtu_core_top.sv index 33452de81b..f4405346dd 100644 --- a/hw/unittest/rtu_core/VX_rtu_core_top.sv +++ b/hw/unittest/rtu_core/VX_rtu_core_top.sv @@ -31,7 +31,7 @@ module VX_rtu_core_top import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Channel widths (mirror VX_rtu_bus_slice / VX_rtu_bus_if). parameter SRC_WIDTH = `UP(`CLOG2(NUM_SRCS)), parameter ARM_DATAW = SRC_WIDTH + NW_WIDTH + NUM_LANES - + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + 32 + TAG_WIDTH, + + `VX_CFG_MEM_ADDR_WIDTH + 16 + 16 + `VX_CFG_XLEN + TAG_WIDTH, parameter REQ_DATAW = 1 + SRC_WIDTH + NW_WIDTH + NUM_LANES * 32 + NUM_LANES * RTU_CB_ACTION_BITS, parameter WIN_DATAW = 1 + NW_WIDTH + RTU_SLOT_BITS diff --git a/miscs/patches/riscv-benchmarks.patch b/miscs/patches/riscv-benchmarks.patch index 3f4011d2c7..4db8b67a94 100644 --- a/miscs/patches/riscv-benchmarks.patch +++ b/miscs/patches/riscv-benchmarks.patch @@ -120,7 +120,7 @@ index 2a2a8f3..e967138 100644 - syscall(SYS_write, HTIF_DEV_CONSOLE, (uintptr_t)&c, HTIF_CONSOLE_CMD_PUTC); - + unsigned tid = read_csr(mhartid) & (VX_MEM_IO_COUT_SLOTS - 1); -+ *(volatile char*)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; ++ *(volatile char*)(uintptr_t)(VX_MEM_IO_COUT_ADDR + tid) = (char)ch; return 0; } diff --git a/profiles/a100.json b/profiles/a100.json new file mode 100644 index 0000000000..7a6003fe0b --- /dev/null +++ b/profiles/a100.json @@ -0,0 +1,35 @@ +{ + "name": "a100", + "description": "NVIDIA A100 SXM4 (GA100, Ampere, sm_80) at its 1410 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 108, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1410, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 1555000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 3038, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 190, + "VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE": 257 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", + "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", + "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + } +} diff --git a/profiles/h100.json b/profiles/h100.json new file mode 100644 index 0000000000..d95d2fabda --- /dev/null +++ b/profiles/h100.json @@ -0,0 +1,32 @@ +{ + "name": "h100", + "description": "NVIDIA H100 SXM5 (GH100, Hopper, sm_90) at its 1980 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 262144 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 132, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 33554432, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1980, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 3350000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM3", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 6545 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + } +} diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json new file mode 100644 index 0000000000..73908bb3d2 --- /dev/null +++ b/profiles/rtx1080.json @@ -0,0 +1,32 @@ +{ + "name": "rtx1080", + "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 32768 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 20, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1607, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", + "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" + } +} diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json new file mode 100644 index 0000000000..d90b047d99 --- /dev/null +++ b/profiles/rtx5080.json @@ -0,0 +1,63 @@ +{ + "name": "rtx5080", + "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 48, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_DCACHE_MSHR_SIZE": 24 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 84, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 67108864, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_L2_MSHR_SIZE": 192 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 2295, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, + "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 + }, + "reference": { + "clock_mhz": 2295, + "l1_hit_cycles": 44.1, + "l2_hit_cycles": 354.4, + "dram_cycles": 802, + "dram_peak_bytes_per_cycle": 423, + "stream_efficiency": { + "4MiB": 0.795, + "16MiB": 0.894, + "64MiB": 0.919 + }, + "sgemm_cycles": { + "128": 24193, + "256": 45476, + "512": 270396, + "1024": 1845287 + }, + "stream_cycles": { + "4MiB": 25569, + "16MiB": 90741, + "64MiB": 427878 + } + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on an RTX 5080)", + "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", + "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", + "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", + "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", + "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", + "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", + "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" + } +} diff --git a/profiles/tinygpu.json b/profiles/tinygpu.json new file mode 100644 index 0000000000..9c496a6237 --- /dev/null +++ b/profiles/tinygpu.json @@ -0,0 +1,15 @@ +{ + "name": "tinygpu", + "description": "The smallest Vortex: one core of 2 warps x 2 threads with shared memory.", + "core": { + "VX_CFG_NUM_WARPS": 2, + "VX_CFG_NUM_THREADS": 2 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 1 + }, + "sources": { + "all": "A minimal reference machine for small FPGAs and fast simulation (1 core, NW=NT=2, shared memory at its default size); the ISA keeps the defaults (RV32IMAF/RV64IMAFD) so it runs the shipped toolchain runtime" + } +} diff --git a/profiles/v100.json b/profiles/v100.json new file mode 100644 index 0000000000..451551d910 --- /dev/null +++ b/profiles/v100.json @@ -0,0 +1,32 @@ +{ + "name": "v100", + "description": "NVIDIA Tesla V100 SXM2 (GV100, Volta, sm_70) at its 1530 MHz boost clock: one Vortex core per SM.", + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_DCACHE_SIZE": 131072 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 80, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 4194304, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 32 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, + "VX_CFG_PLATFORM_CLOCK_RATE": 1530, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 900000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "HBM2", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 1752 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "80 SMs, 64 resident warps/SM, 4 processing blocks with one scheduler each (NVIDIA Tesla V100 GPU Architecture whitepaper, Table 1)", + "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + } +} diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 539c3922ef..dfe8ab9bb4 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -10,8 +10,8 @@ XRTSIM_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/aved AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -25,7 +25,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE @@ -213,8 +213,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 85083ef49d..0972440954 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -26,6 +26,9 @@ constexpr unsigned cp_clog2(uint64_t n) { } uint64_t gpu_dev_caps() { const unsigned cluster_size = VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE; + static_assert(VX_CFG_NUM_WARPS < 128 && VX_CFG_NUM_CLUSTERS < 128 + && VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE < 256, + "GPU_DEV_CAPS exact-count fields overflow"); const unsigned bank_addr_w = VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); return (uint64_t(VX_ISA_IMPL_ID) & 0xFF) @@ -37,7 +40,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cp_clog2(VX_CFG_ISSUE_WIDTH)) & 0x7) << 23) | ((uint64_t(VX_CFG_LMEM_ENABLED ? VX_CFG_LMEM_LOG_SIZE : 0) & 0xFF) << 26) | ((uint64_t(cp_clog2(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)) & 0x7) << 34) - | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37); + | ((uint64_t(bank_addr_w - 20) & 0x1F) << 37) + | ((uint64_t(VX_CFG_NUM_WARPS) & 0x7F) << 42) + | ((uint64_t(cluster_size) & 0xFF) << 49) + | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index cbb881e713..bc899b89fc 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -13,7 +13,14 @@ #include "dram_sim.h" #include "util.h" +#include +#include #include +#include +#include +#include +#include +#include DISABLE_WARNING_PUSH DISABLE_WARNING_UNUSED_PARAMETER @@ -27,146 +34,352 @@ DISABLE_WARNING_POP using namespace vortex; +namespace { + +// Bus timings are DRAM clocks, fixed by the burst; array timings are +// nanoseconds, which Ramulator converts at the configured data rate. +struct clock_timing_t { + const char* name; + int clocks; +}; + +struct ns_timing_t { + const char* name; + float ns; +}; + +// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same +// spacing between column commands to another bank group (four within one). +// nRTW only completes the set Ramulator requires. +const clock_timing_t hbm_bus_timing[] = { + {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, +}; + +// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its +// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); +// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its +// cycles nanoseconds. The array does not speed up with the interface: HBM2E +// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM +// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete +// the set Ramulator requires. +const ns_timing_t hbm_array_timing[] = { + {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, + {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, +}; + +// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a +// x16 channel, is eight clocks, and column commands are two (three within a +// bank group) command clocks apart, a command clock being eight data beats. +const clock_timing_t gddr6_bus_timing[] = { + {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, +}; + +// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset +// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh +// timings complete the set Ramulator requires. +const ns_timing_t gddr6_array_timing[] = { + {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, + {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, + {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, + {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, +}; + +template +void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A]) { + timing["rate"] = data_rate; + for (auto& t : bus) { + timing[t.name] = t.clocks; + } + for (auto& t : array) { + timing[t.name] = t.ns; + } +} + +} + class DramSim::Impl { private: - struct mem_req_t { - uint64_t addr; - bool is_write; - ResponseCallback callback; - void* arg; - }; - - struct mem_rsp_t { - ResponseCallback callback; - void* arg; - }; - - Ramulator::IFrontEnd* ramulator_frontend_; - Ramulator::IMemorySystem* ramulator_memorysystem_; - uint32_t cpu_channel_size_; - uint64_t cpu_cycles_; - uint32_t scaled_dram_cycles_; - static const uint32_t tick_cycles_ = 1000; - static const uint32_t dram_channel_size_ = 16; // 128 bits - std::queue pending_reqs_; - std::queue pending_rsps_; - - void handle_pending_responses() { - if (pending_rsps_.empty()) - return; - auto& rsp = pending_rsps_.front(); - if (rsp.callback(rsp.arg)) { - pending_rsps_.pop(); - } - } - - void handle_pending_requests() { - if (pending_reqs_.empty()) - return; - auto& req = pending_reqs_.front(); - auto req_type = req.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; - std::function callback = nullptr; - if (req.callback) { - callback = [this, req_callback = std::move(req.callback), req_arg = std::move(req.arg)](Ramulator::Request& /*dram_req*/) { - this->pending_rsps_.push({req_callback, req_arg}); - }; - } - if (ramulator_frontend_->receive_external_requests(req_type, req.addr, 0, callback)) { - // Ramulator does not handle write responses, so we handle the callback ourselves. - if (req.is_write && req.callback) { - pending_rsps_.push({req.callback, req.arg}); - } - pending_reqs_.pop(); - } - } + // One request (a block); it completes when all its transactions have. + struct request_t { + ResponseCallback callback; + void* arg; + uint32_t pending; + }; + + struct mem_tx_t { + uint64_t addr; + bool is_write; + std::shared_ptr request; + }; + + struct mem_rsp_t { + std::shared_ptr request; + uint64_t ready_cycle; + }; + + // An independent controller and its device. It accepts one transaction per + // DRAM cycle and returns its responses independently of the other channels; + // write acknowledgements do not wait behind read data. + struct channel_t { + Ramulator::IFrontEnd* frontend; + Ramulator::IMemorySystem* memory_system; + std::queue pending_txs; + std::deque read_rsps; + std::deque write_rsps; + uint32_t inflight; + }; + + // Each standard moves 32 B per column command (prefetch x channel width). + static constexpr uint32_t tx_size_ = 32; + + // Every device here holds 8 Gb per channel; addresses above alias, which + // only affects row hits. + static constexpr uint64_t channel_size_ = uint64_t(1) << 30; + + // Bursts per row visit (minimalist open page, 128 B). + static constexpr uint32_t row_visit_bits_ = 2; + + Config config_; + uint32_t bank_bits_; // pseudo-channel, bank group and bank + uint32_t column_bits_; // bursts per row + std::vector channels_; + uint64_t cycle_; + // DRAM ticks owed, in units of 1/(2 x core_clock): each core cycle adds the + // data rate, each DRAM tick (half the data rate) takes 2 x core_clock. + uint64_t clock_credit_; + + static void set_hbm_device(YAML::Node dram, const Config& config) { + // Ramulator's 8 Gb organization: 2 pseudo-channels x 4 bank groups x 4 + // banks, 1 KB rows; a 128-bit channel width with the 2n prefetch makes a + // transaction one 32 B pseudo-channel burst. + dram["impl"] = (config.type == Type::HBM3) ? "HBM3" : "HBM2"; + dram["org"]["preset"] = (config.type == Type::HBM3) ? "HBM3_8Gb" : "HBM2_8Gb"; + dram["org"]["density"] = 8192; + dram["org"]["channel"] = 1; + dram["org"]["channel_width"] = 128; + set_timing(dram["timing"], config.data_rate, hbm_bus_timing, hbm_array_timing); + } + + static void set_gddr6_device(YAML::Node dram, const Config& config) { + // One x16 channel of a 16 Gb device (8 Gb), 4 bank groups of 4 banks, + // 2 KB pages. A 32 B burst is 16n prefetch on 16 pins; Ramulator's GDDR6 + // prefetches 8 columns, so its columns are 32 bits. + dram["impl"] = "GDDR6"; + dram["org"]["dq"] = 32; + dram["org"]["channel"] = 1; + dram["org"]["bankgroup"] = 4; + dram["org"]["bank"] = 4; + dram["org"]["row"] = 1 << 15; + dram["org"]["column"] = 512; + dram["org"]["density"] = 8192; + dram["org"]["channel_width"] = 32; + set_timing(dram["timing"], config.data_rate, gddr6_bus_timing, gddr6_array_timing); + } + + YAML::Node channel_config(uint32_t ch) const { + YAML::Node config; + config["Frontend"]["impl"] = "GEM5"; + auto memory_system = config["MemorySystem"]; + memory_system["impl"] = "GenericDRAM"; + memory_system["clock_ratio"] = 1; + if (config_.type == Type::GDDR6) { + set_gddr6_device(memory_system["DRAM"], config_); + } else { + set_hbm_device(memory_system["DRAM"], config_); + } + auto controller = memory_system["Controller"]; + controller["impl"] = "Generic"; + controller["Scheduler"]["impl"] = "FRFCFS"; + // No refresh is modelled: Ramulator's refresh manager refreshes per rank + // and these devices have no rank level. GPU controllers hide refresh with + // per-bank refresh, for which Ramulator has no manager (and its HBM REFsb + // lacks the bank's tRFCsb busy time); all-bank refresh would cost every + // channel ~9% of its time instead. + controller["RefreshManager"]["impl"] = "AllBank"; + controller["RowPolicy"]["impl"] = "OpenRowPolicy"; + // Per-command DRAM trace recording is a debugging aid only: it writes an + // entry for every DRAM command and dominates simulation time (100s of MB + // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env + // var so default simx/rtlsim runs stay fast. + if (const char* trace_path = getenv("DRAM_TRACE")) { + std::string path = trace_path[0] ? trace_path : "./trace/ramulator.log"; + YAML::Node trace_plugin; + trace_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; + trace_plugin["ControllerPlugin"]["path"] = path + "." + std::to_string(ch); + controller["plugins"].push_back(trace_plugin); + } + // Fields in the order dram_addr lays them out. + memory_system["AddrMapper"]["impl"] = "RoBaRaCoCh"; + return config; + } + + // Ramulator address of a transaction. A row visit is four bursts, then the + // next pseudo-channel, bank group and bank (minimalist open page), and the + // bank is XORed with the row folded to the bank's width (permutation-based + // interleaving), so streams a power of two apart open rows in different + // banks. + uint64_t dram_addr(uint64_t addr) const { + uint64_t tx = (addr & (channel_size_ - 1)) / tx_size_; + uint64_t visit = tx & ((1u << row_visit_bits_) - 1); + tx >>= row_visit_bits_; + uint64_t bank = tx & ((1u << bank_bits_) - 1); + tx >>= bank_bits_; + uint64_t column = tx & ((1u << (column_bits_ - row_visit_bits_)) - 1); + uint64_t row = tx >> (column_bits_ - row_visit_bits_); + for (uint64_t r = row; r != 0; r >>= bank_bits_) { + bank ^= r & ((1u << bank_bits_) - 1); + } + uint64_t t = (row << bank_bits_) | bank; + t = (t << column_bits_) | (column << row_visit_bits_) | visit; + return t * tx_size_; + } + + // Retires the ready head responses of `rsps`; a request completes with its + // last transaction. + void drain(std::deque& rsps, channel_t& channel) { + while (!rsps.empty()) { + auto& rsp = rsps.front(); + if (rsp.ready_cycle > cycle_) { + break; + } + auto& request = *rsp.request; + if (request.pending == 1 && request.callback && !request.callback(request.arg)) { + break; + } + if (--request.pending == 0) { + --channel.inflight; + } + rsps.pop_front(); + } + } + + void issue(channel_t& channel) { + if (channel.pending_txs.empty()) { + return; + } + auto& tx = channel.pending_txs.front(); + auto req_type = tx.is_write ? Ramulator::Request::Type::Write : Ramulator::Request::Type::Read; + std::function callback = nullptr; + if (!tx.is_write) { + callback = [this, &channel, request = tx.request](Ramulator::Request& /*dram_req*/) { + channel.read_rsps.push_back({request, this->cycle_ + 1 + this->config_.latency}); + }; + } + if (channel.frontend->receive_external_requests(req_type, tx.addr, 0, callback)) { + // Ramulator does not handle write responses, so we handle the callback ourselves. + if (tx.is_write) { + channel.write_rsps.push_back({tx.request, cycle_}); + } + channel.pending_txs.pop(); + } + } public: - Impl(uint32_t num_channels, uint32_t channel_size, float clock_ratio) { - YAML::Node dram_config; - dram_config["Frontend"]["impl"] = "GEM5"; - dram_config["MemorySystem"]["impl"] = "GenericDRAM"; - dram_config["MemorySystem"]["clock_ratio"] = 1; - dram_config["MemorySystem"]["DRAM"]["impl"] = "HBM2"; - dram_config["MemorySystem"]["DRAM"]["org"]["preset"] = "HBM2_8Gb"; - dram_config["MemorySystem"]["DRAM"]["org"]["density"] = 8192; - dram_config["MemorySystem"]["DRAM"]["org"]["channel"] = num_channels; - dram_config["MemorySystem"]["DRAM"]["timing"]["preset"] = "HBM2_2Gbps"; - dram_config["MemorySystem"]["Controller"]["impl"] = "Generic"; - dram_config["MemorySystem"]["Controller"]["Scheduler"]["impl"] = "FRFCFS"; - dram_config["MemorySystem"]["Controller"]["RefreshManager"]["impl"] = "AllBank"; - dram_config["MemorySystem"]["Controller"]["RowPolicy"]["impl"] = "OpenRowPolicy"; - // Per-command DRAM trace recording is a debugging aid only: it writes an - // entry for every DRAM command and dominates simulation time (100s of MB - // of I/O for even tiny workloads). Keep it opt-in via the DRAM_TRACE env - // var so default simx/rtlsim runs stay fast. - if (const char* trace_path = getenv("DRAM_TRACE")) { - YAML::Node draw_plugin; - draw_plugin["ControllerPlugin"]["impl"] = "TraceRecorder"; - draw_plugin["ControllerPlugin"]["path"] = trace_path[0] ? trace_path : "./trace/ramulator.log"; - dram_config["MemorySystem"]["Controller"]["plugins"].push_back(draw_plugin); - } - dram_config["MemorySystem"]["AddrMapper"]["impl"] = "RoBaRaCoCh"; - - ramulator_frontend_ = Ramulator::Factory::create_frontend(dram_config); - ramulator_memorysystem_ = Ramulator::Factory::create_memory_system(dram_config); - ramulator_frontend_->connect_memory_system(ramulator_memorysystem_); - ramulator_memorysystem_->connect_frontend(ramulator_frontend_); - - cpu_channel_size_ = channel_size; - scaled_dram_cycles_ = static_cast(clock_ratio * tick_cycles_); - this->reset(); - } - - ~Impl() { - std::ofstream nullstream("ramulator.stats.log"); - auto original_buf = std::cout.rdbuf(); - std::cout.rdbuf(nullstream.rdbuf()); - ramulator_frontend_->finalize(); - ramulator_memorysystem_->finalize(); - std::cout.rdbuf(original_buf); - } - - void reset() { - cpu_cycles_ = 0; - pending_reqs_ = std::queue(); - pending_rsps_ = std::queue(); - } - - void tick() { - cpu_cycles_ += tick_cycles_; - this->handle_pending_responses(); - while (cpu_cycles_ >= scaled_dram_cycles_) { - this->handle_pending_requests(); - ramulator_memorysystem_->tick(); - cpu_cycles_ -= scaled_dram_cycles_; - } - } - - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { - if (cpu_channel_size_ > dram_channel_size_) { - uint32_t n = cpu_channel_size_ / dram_channel_size_; - for (uint32_t i = 0; i < n; ++i) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_ + (i * dram_channel_size_); - if (i == 0) { - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - pending_reqs_.push({dram_byte_addr, is_write, nullptr, nullptr}); - } - } - } else if (cpu_channel_size_ < dram_channel_size_) { - uint64_t dram_byte_addr = (addr / cpu_channel_size_) * dram_channel_size_; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } else { - uint64_t dram_byte_addr = addr; - pending_reqs_.push({dram_byte_addr, is_write, response_cb, arg}); - } - } + Impl(const Config& config) + : config_(config) + // HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 32 bursts per 1 KB + // row; GDDR6: 4 x 4 banks, 64 bursts per 2 KB row. + , bank_bits_((config.type == Type::GDDR6) ? 4 : 5) + , column_bits_((config.type == Type::GDDR6) ? 6 : 5) + , channels_(config.num_channels) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + auto ramulator_config = this->channel_config(ch); + auto& channel = channels_[ch]; + channel.frontend = Ramulator::Factory::create_frontend(ramulator_config); + channel.memory_system = Ramulator::Factory::create_memory_system(ramulator_config); + channel.frontend->connect_memory_system(channel.memory_system); + channel.memory_system->connect_frontend(channel.frontend); + } + this->reset(); + } + + ~Impl() { + std::ofstream nullstream("ramulator.stats.log"); + auto original_buf = std::cout.rdbuf(); + std::cout.rdbuf(nullstream.rdbuf()); + for (auto& channel : channels_) { + channel.frontend->finalize(); + channel.memory_system->finalize(); + } + std::cout.rdbuf(original_buf); + } + + void reset() { + cycle_ = 0; + clock_credit_ = 0; + for (auto& channel : channels_) { + channel.pending_txs = std::queue(); + channel.read_rsps.clear(); + channel.write_rsps.clear(); + channel.inflight = 0; + } + } + + void tick() { + ++cycle_; + clock_credit_ += config_.data_rate; + for (auto& channel : channels_) { + this->drain(channel.read_rsps, channel); + this->drain(channel.write_rsps, channel); + } + while (clock_credit_ >= 2 * config_.core_clock) { + for (auto& channel : channels_) { + this->issue(channel); + channel.memory_system->tick(); + } + clock_credit_ -= 2 * config_.core_clock; + } + } + + bool can_accept(uint32_t channel) const { + return (config_.queue_size == 0) || (channels_.at(channel).inflight < config_.queue_size); + } + + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { + // A block wider than a burst is several transactions. + uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; + uint64_t base = addr & ~uint64_t(config_.block_size - 1); + auto request = std::make_shared(request_t{response_cb, arg, n}); + auto& ch = channels_.at(channel); + for (uint32_t i = 0; i < n; ++i) { + ch.pending_txs.push({dram_addr(base + i * tx_size_), is_write, request}); + } + ++ch.inflight; + } }; /////////////////////////////////////////////////////////////////////////////// -DramSim::DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio) - : impl_(new Impl(num_channels, channel_size, clock_ratio)) +#define DRAM_STR_(x) #x +#define DRAM_STR(x) DRAM_STR_(x) + +DramSim::Config DramSim::platform_config() { + // The enum's value macro, not its per-value companion: the simulators get + // raw CONFIGS, where -DVX_CFG_PLATFORM_MEMORY_TYPE= defines no + // companion. + const std::string type = DRAM_STR(VX_CFG_PLATFORM_MEMORY_TYPE); + Config config; + if (type == "HBM2") { + config.type = Type::HBM2; + } else if (type == "HBM3") { + config.type = Type::HBM3; + } else if (type == "GDDR6") { + config.type = Type::GDDR6; + } else { + throw std::invalid_argument("dram_sim: unknown VX_CFG_PLATFORM_MEMORY_TYPE " + type); + } + config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + config.block_size = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + config.core_clock = VX_CFG_PLATFORM_CLOCK_RATE; + config.data_rate = VX_CFG_PLATFORM_MEMORY_DATA_RATE; + config.latency = VX_CFG_PLATFORM_MEMORY_LATENCY; + config.queue_size = VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE; + return config; +} + +DramSim::DramSim(const Config& config) + : impl_(new Impl(config)) {} DramSim::~DramSim() { @@ -181,6 +394,10 @@ void DramSim::tick() { impl_->tick(); } -void DramSim::send_request(uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { - impl_->send_request(addr, is_write, callback, arg); -} \ No newline at end of file +bool DramSim::can_accept(uint32_t channel) const { + return impl_->can_accept(channel); +} + +void DramSim::send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback callback, void* arg) { + impl_->send_request(channel, addr, is_write, callback, arg); +} diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 7e8741791c..565dede377 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -11,27 +11,55 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#pragma once + +#include namespace vortex { +// Device memory behind the platform's memory banks: one DRAM channel (an +// independent controller) per bank, timed by Ramulator. class DramSim { public: typedef bool (*ResponseCallback)(void *arg); - DramSim(uint32_t num_channels, uint32_t channel_size, float clock_ratio); + // The JEDEC standards Ramulator models, each at its own organization and + // timing; the data rate picks the speed. + enum class Type { + HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts + HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages + }; + + struct Config { + Type type; + uint32_t num_channels; // need not be a power of two + uint32_t block_size; // bytes per request + uint32_t core_clock; // MHz + uint32_t data_rate; // MT/s per pin + uint32_t latency; // fixed read latency added in core cycles (PHY, controller, NoC) + uint32_t queue_size; // requests a channel holds in flight; 0 = unbounded + }; + + // The memory configured in VX_config.toml. + static Config platform_config(); + + DramSim(const Config& config); ~DramSim(); void reset(); void tick(); - // addr: per-channel block address - void send_request(uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); + // False while the channel holds queue_size requests. + bool can_accept(uint32_t channel) const; + + // addr: byte address within the channel. + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); private: - class Impl; - Impl* impl_; + class Impl; + Impl* impl_; }; -} \ No newline at end of file +} diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 5fc6e6aaee..f4dbfb723a 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -6,8 +6,8 @@ CONFIG_FILE := $(DESTDIR)/opaesim_config.stamp SRC_DIR := $(VORTEX_HOME)/sim/opaesim AFU_DIR := $(RTL_DIR)/afu/opae -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -21,7 +21,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # debug trace enable flags DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/opaesim/opae_sim.cpp b/sim/opaesim/opae_sim.cpp index 94e9adfbe5..7e37ee5dde 100644 --- a/sim/opaesim/opae_sim.cpp +++ b/sim/opaesim/opae_sim.cpp @@ -40,15 +40,12 @@ #include #include #include +#include #include #include #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #define CACHE_BLOCK_SIZE 64 #define CCI_LATENCY 8 @@ -100,7 +97,7 @@ class opae_sim::Impl { Impl() : device_(nullptr) , ram_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) , host_buffer_ids_(0) #ifdef VCD_OUTPUT @@ -297,9 +294,8 @@ class opae_sim::Impl { reqs.clear(); } - { - std::queue empty; - std::swap(dram_queue_, empty); + for (auto& queue : dram_queues_) { + queue = std::queue(); } device_->reset = 1; @@ -318,9 +314,15 @@ class opae_sim::Impl { this->cci_bus_eval(); this->avs_bus_eval(); - if (!dram_queue_.empty()) { - auto mem_req = dram_queue_.front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.tick(); + + for (uint32_t b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& queue = dram_queues_[b]; + if (queue.empty() || !dram_sim_.can_accept(b)) { + continue; + } + auto mem_req = queue.front(); + dram_sim_.send_request(b, uint64_t(mem_req->addr) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE, mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -329,11 +331,9 @@ class opae_sim::Impl { } return true; }, mem_req); - dram_queue_.pop(); + queue.pop(); } - dram_sim_.tick(); - device_->clk = 0; this->eval(); device_->clk = 1; @@ -499,7 +499,7 @@ class opae_sim::Impl { mem_req->write = true; mem_req->ready = true; - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } else if (device_->avs_read[b]) { // process read request @@ -512,7 +512,7 @@ class opae_sim::Impl { pending_mem_reqs_[b].emplace_back(mem_req); // send dram request - dram_queue_.push(mem_req); + dram_queues_[b].push(mem_req); } device_->avs_waitrequest[b] = false; @@ -580,7 +580,7 @@ class opae_sim::Impl { } }; - std::queue dram_queue_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> dram_queues_; #ifdef VCD_OUTPUT VerilatedVcdC *tfp_; diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 9856159b26..f15c7da8a7 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -5,8 +5,8 @@ DESTDIR ?= $(CURDIR) CONFIG_FILE = $(DESTDIR)/rtlsim_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/rtlsim -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -20,7 +20,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE @@ -132,9 +132,7 @@ TOP = rtlsim_shim VL_FLAGS = --exe VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -# Keep lint warnings visible but non-fatal: newer Verilator surfaces latent -# UNUSED*/PINCONNECTEMPTY warnings in existing IP that older releases did not. -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -201,11 +199,13 @@ PROJECT := rtlsim all: $(DESTDIR)/$(PROJECT) +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SRC_DIR)/main.cpp $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) $(SRC_DIR)/main.cpp -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ $(DESTDIR)/lib$(PROJECT).so: $(SRCS) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build $(VL_FLAGS) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '-shared $(LDFLAGS)' --MMD --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index a0e223f2e0..b383d556ff 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -48,9 +48,6 @@ #include #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull @@ -101,7 +98,7 @@ bool sim_trace_enabled() { class Processor::Impl { public: - Impl() : dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) { + Impl() : dram_sim_(DramSim::platform_config()) { // restart sim time so a re-created model registers at time zero timestamp = 0; @@ -306,9 +303,9 @@ class Processor::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queue_[b].empty()) { + if (!dram_queue_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queue_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, mem_req->bank_addr, mem_req->write, [](void* arg)->bool { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; @@ -400,6 +397,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = true; mem_req->ready = false; @@ -414,6 +412,7 @@ class Processor::Impl { auto mem_req = new mem_req_t(); mem_req->tag = device_->mem_req_tag[b]; mem_req->addr = byte_addr; + mem_req->bank_addr = uint64_t(device_->mem_req_addr[b]) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; mem_req->write = false; mem_req->ready = false; ram_->read(mem_req->data.data(), byte_addr, VX_CFG_PLATFORM_MEMORY_DATA_SIZE); @@ -445,6 +444,7 @@ class Processor::Impl { Vrtlsim_shim* device; std::array data; uint64_t addr; + uint64_t bank_addr; uint64_t tag; bool write; bool ready; diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 63581b4b8f..746bd6f7f8 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -17,7 +17,7 @@ CONFIG_FILE = $(DESTDIR)/simx_config.stamp SRC_DIR = $(VORTEX_HOME)/sim/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)/dxa -I$(SRC_DIR)/tex -I$(SRC_DIR)/om -I$(SRC_DIR)/raster -I$(SRC_DIR)/rtu -I$(SRC_DIR)/kmu -I$(SRC_DIR)/mem -I$(SRC_DIR)/dtm -I$(SW_COMMON_DIR) -I$(SIM_COMMON_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(THIRD_PARTY_DIR) CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include CXXFLAGS += -I$(THIRD_PARTY_DIR)/ramulator/ext/spdlog/include @@ -27,7 +27,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 1bdfce56a5..2ef860870a 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -20,10 +20,6 @@ #define RAM_PAGE_SIZE 4096 #endif -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif - #ifndef __MAX #define __MAX(a,b) (((a) > (b)) ? (a) : (b)) #endif diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index f67dba6e23..ec729d83e3 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -69,7 +69,7 @@ class Core::Impl { , fetch_latch_(ctx, "fetch_latch", 2, 2) , decode_latch_(ctx, "decode_latch", 1, 2) , pending_icache_(VX_CFG_NUM_WARPS) - , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GTO, PER_ISSUE_WARPS}) + , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index 41ed3b411a..8c4dcfc0a8 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -18,6 +18,7 @@ #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif +#include #include #include #include @@ -649,7 +650,7 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency)), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index b3ffe9c8be..70ad9e6e3f 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -35,12 +35,12 @@ class Cache : public SimObject { uint8_t A; // log2 associativity uint8_t B; // log2 number of banks uint8_t addr_width; // word address bits - uint8_t num_inputs; // number of inputs + uint16_t num_inputs; // number of inputs uint8_t mem_ports; // memory ports bool write_back; // is write-back bool write_reponse; // enable write response uint16_t mshr_size; // MSHR buffer size - uint8_t latency; // pipeline latency + uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache }; diff --git a/sim/simx/mem/memory.cpp b/sim/simx/mem/memory.cpp index 74c3782edc..50db7006ae 100644 --- a/sim/simx/mem/memory.cpp +++ b/sim/simx/mem/memory.cpp @@ -55,13 +55,13 @@ class Memory::Impl { Impl(Memory* simobject, const Config& config) : simobject_(simobject) , config_(config) - , dram_sim_(config.num_banks, config.block_size, config.clock_ratio) + , dram_sim_(config.dram) , ram_(nullptr) { char sname[100]; snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); - mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.num_banks, - [lg2_block_size = log2ceil(config.block_size), num_banks = config.num_banks](const MemCrossBar::ReqType& req) { + mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config.num_ports, config.dram.num_channels, + [lg2_block_size = log2ceil(config.dram.block_size), num_banks = config.dram.num_channels](const MemCrossBar::ReqType& req) { // Custom logic to calculate the output index using bank interleaving return (uint32_t)((req.addr >> lg2_block_size) & (num_banks-1)); }); @@ -85,8 +85,10 @@ class Memory::Impl { void tick() { dram_sim_.tick(); - for (uint32_t i = 0; i < config_.num_banks; ++i) { - if (mem_xbar_->ReqOut.at(i).empty()) + uint32_t num_banks = config_.dram.num_channels; + uint32_t lg2_block_size = log2ceil(config_.dram.block_size); + for (uint32_t i = 0; i < num_banks; ++i) { + if (mem_xbar_->ReqOut.at(i).empty() || !dram_sim_.can_accept(i)) continue; auto& mem_req = mem_xbar_->ReqOut.at(i).peek(); @@ -126,8 +128,10 @@ class Memory::Impl { // enqueue the request to the memory system auto req_args = new DramCallbackArgs{this, mem_req, i, rsp_data}; + uint64_t bank_addr = ((mem_req.addr >> lg2_block_size) / num_banks) << lg2_block_size; dram_sim_.send_request( - mem_req.addr, + i, + bank_addr, mem_req.is_write(), [](void* arg)->bool { auto rsp_args = reinterpret_cast(arg); diff --git a/sim/simx/mem/memory.h b/sim/simx/mem/memory.h index ea060c0e05..dd4a95be39 100644 --- a/sim/simx/mem/memory.h +++ b/sim/simx/mem/memory.h @@ -15,6 +15,7 @@ #include "types.h" #include +#include #include namespace vortex { @@ -22,10 +23,8 @@ namespace vortex { class Memory : public SimObject{ public: struct Config { - uint32_t num_banks; uint32_t num_ports; - uint32_t block_size; - float clock_ratio; + DramSim::Config dram; // one DRAM channel per memory bank }; struct PerfStats { diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 0aa304656f..52ad76ca1a 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -51,12 +51,9 @@ ProcessorImpl::ProcessorImpl() kmu_ = Kmu::Create("kmu", total_cores); // create memory simulator - memsim_ = Memory::Create("dram", Memory::Config{ - VX_CFG_PLATFORM_MEMORY_NUM_BANKS, - VX_CFG_L3_MEM_PORTS, - VX_CFG_MEM_BLOCK_SIZE, - MEM_CLOCK_RATIO - }); + auto dram_config = DramSim::platform_config(); + dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); char sname[100]; diff --git a/sim/simx/rtu/rtu_core.cpp b/sim/simx/rtu/rtu_core.cpp index 8c2872fdb9..b8f53f321e 100644 --- a/sim/simx/rtu/rtu_core.cpp +++ b/sim/simx/rtu/rtu_core.cpp @@ -487,14 +487,14 @@ class RtuCore::Impl { } void bind_context(uint32_t ci, uint32_t slot, uint32_t lane, - uint32_t scene_root) { + uint64_t scene_root) { Context& cx = contexts_[ci]; cx = Context{}; cx.valid = true; cx.slot = slot; cx.lane = lane; - cx.base_line = uint64_t(scene_root) & kRtuLineMask; - cx.byte_off = uint32_t(uint64_t(scene_root) - cx.base_line); + cx.base_line = scene_root & kRtuLineMask; + cx.byte_off = uint32_t(scene_root - cx.base_line); // The ray opens on the reciprocal (1/dir) setup and the scene-header fetch, // which cost front-end states like any other step, then drain the reciprocal // pipeline before the first node can be tested. diff --git a/sim/simx/rtu/rtu_raylog.cpp b/sim/simx/rtu/rtu_raylog.cpp index 4281972b50..42592e10ec 100644 --- a/sim/simx/rtu/rtu_raylog.cpp +++ b/sim/simx/rtu/rtu_raylog.cpp @@ -158,13 +158,13 @@ class Logger { static constexpr uint64_t kInstTableSpan = 64 * 1024; static constexpr uint64_t kMaxSceneBytes = 1ull << 30; - void snapshot_scene(uint32_t root) { + void snapshot_scene(uint64_t root) { if (ram_ == nullptr || full() || !snapped_.insert(root).second) return; const RAM& ram = *ram_; uint32_t scene_bytes = 0; for (int i = 0; i < 4; ++i) scene_bytes |= uint32_t(ram[root + 8 + i]) << (8 * i); const uint64_t lo = (root > kInstTableSpan ? root - kInstTableSpan : 0) & kRtuLineMask; - const uint64_t hi = uint64_t(root) + std::min(scene_bytes, kMaxSceneBytes); + const uint64_t hi = root + std::min(scene_bytes, kMaxSceneBytes); uint64_t added = 0; for (uint64_t a = lo; a < hi; a += VX_CFG_MEM_BLOCK_SIZE) { if (image_.count(a)) continue; @@ -184,12 +184,12 @@ class Logger { std::fwrite(&r, sizeof(r), 1, fp_); ++added; } - std::fprintf(stderr, "[rtu-raylog] scene 0x%x: %u bytes, snapshot %llu lines\n", - root, scene_bytes, (unsigned long long)added); + std::fprintf(stderr, "[rtu-raylog] scene 0x%llx: %u bytes, snapshot %llu lines\n", + (unsigned long long)root, scene_bytes, (unsigned long long)added); } const RAM* ram_ = nullptr; - std::unordered_set snapped_; + std::unordered_set snapped_; FILE* fp_ = nullptr; unsigned long long max_rays_ = 4ull << 20; unsigned long long every_ = 1; diff --git a/sim/simx/rtu/rtu_raylog.h b/sim/simx/rtu/rtu_raylog.h index d9b284cee1..b73b2d834b 100644 --- a/sim/simx/rtu/rtu_raylog.h +++ b/sim/simx/rtu/rtu_raylog.h @@ -47,7 +47,7 @@ class RAM; namespace rtu { namespace raylog { constexpr uint32_t kMagic = 0x4C525856; // "VXRL" -constexpr uint32_t kVersion = 1; +constexpr uint32_t kVersion = 2; enum RecType : uint32_t { REC_LINE = 1, @@ -80,7 +80,7 @@ struct RaylogRay { uint32_t info; uint32_t seq; uint32_t epoch; - uint32_t scene_root; + uint64_t scene_root; uint32_t ray_flags; uint32_t cull_mask; uint32_t warp_lane; // lane | warp << 8 | slot << 16 @@ -97,7 +97,7 @@ struct RaylogRay { uint32_t inst_id; uint32_t inst_custom; uint32_t hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index 07d0ecb695..8e84ba11cb 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -66,7 +66,7 @@ struct RtuReq { uint32_t slot_idx = 0; // Per-lane ray descriptor snapshot (TRACE_NEW only). - std::array scene_root = {}; + std::array scene_root = {}; std::array origin_x = {}; std::array origin_y = {}; std::array origin_z = {}; diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 5b6a93ad8a..2d8cef624f 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -75,6 +75,10 @@ namespace { inline uint32_t fp_src_bits(const instr_trace_t* trace, uint32_t src, uint32_t t) { return static_cast(trace->src_data[src].at(t).u64 & 0xffffffffu); } + +// The scene base rides the arm doorbell as a device address, MEM_ADDR_WIDTH wide. +constexpr uint64_t kSceneAddrMask = (VX_CFG_MEM_ADDR_WIDTH >= 64) + ? ~uint64_t(0) : ((uint64_t(1) << VX_CFG_MEM_ADDR_WIDTH) - 1); } // namespace instr_trace_t* RtuUnit::process_wait(instr_trace_t* trace, uint32_t block_id) { @@ -260,7 +264,7 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { auto& lregs = wregs.at(t); // Written by the RTU, not by any instruction: the payload pointer rode the // arm and the hitAttribute rode the CONTINUE that accepted this hit. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -280,6 +284,14 @@ void RtuUnit::apply_response(const RtuRsp& rsp) { } } +void RtuUnit::stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const { + uint64_t payload = trace_payload_.at(wid); + lregs[VX_RT_PAYLOAD_PTR_LO] = uint32_t(payload); +#ifdef VX_CFG_XLEN_64 + lregs[VX_RT_PAYLOAD_PTR_LO + 1] = uint32_t(payload >> 32); +#endif +} + void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // Stage candidate-hit attrs + cb_type + cb_handle into the RTU regs // for the lanes whose rays yielded, so the dispatcher's vx_rt_get @@ -295,7 +307,7 @@ void RtuUnit::apply_callback_payload(const RtuRsp& rsp) { // The callback shaders read the tracing shader's payload pointer, and they // are a different shader — it cannot reach them any other way. A closest-hit // shader likewise reads the accepted hit's attribute. - lregs[VX_RT_PAYLOAD_PTR_LO] = trace_payload_.at(rsp.warp_id); + stage_payload_ptr(lregs, rsp.warp_id); lregs[VX_RT_HIT_ATTR_0] = rsp.hit_attr[t]; lregs[VX_RT_HIT_T] = float_to_bits(rsp.hit_t[t]); lregs[VX_RT_HIT_BARY_U] = float_to_bits(rsp.hit_bary_u[t]); @@ -437,7 +449,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i case 0: { // GP config uop: the pool slot was claimed at issue (trace2_reserve_slot), // so this uop has no backpressure source. Unpack the lane-packed config - // (lane0=scene, lane1=payload, lane2=flags, lane3=cull — the implicit + // (lane1=scene, lane2=payload, lane3=flags|cull — the implicit // vx_wgather layout) and stage it. int32_t slot = trace_slot_.at(wid); assert(slot >= 0 && "TRACE uop0 issued without a reserved pool slot"); @@ -448,12 +460,12 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i uint32_t flagscull = static_cast(cfg.at(3).u); // Warp-uniform: it rides the arm doorbell, so it is staged here, not written // into the window. The RTU writes the payload pointer back with the record. - trace_payload_.at(wid) = static_cast(cfg.at(2).u); + trace_payload_.at(wid) = cfg.at(2).u; trace_flags_.at(wid) = flagscull & 0xffffu; trace_cull_.at(wid) = flagscull >> 16; for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) { if (!trace->tmask.test(t)) continue; - trace_scene_.at(wid)[t] = static_cast(cfg.at(1).u); + trace_scene_.at(wid)[t] = uint64_t(cfg.at(1).u) & kSceneAddrMask; trace->dst_data[t].u = uint32_t(slot); // handle returns early } return trace; diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index ab807f324c..978e7ea511 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -156,6 +156,9 @@ class RtuUnit { // code reads the right payload before issuing CONTINUE. void apply_callback_payload(const RtuRsp& rsp); + // The payload pointer is XLEN wide; the window holds it as 32-bit words. + void stage_payload_ptr(RtuWindow::LaneRegs& lregs, uint32_t wid) const; + // Representative slot of a candidate rsp (first active lane's cb_handle). static uint32_t candidate_slot(const RtuRsp& rsp); @@ -225,9 +228,9 @@ class RtuUnit { VX_CFG_NUM_WARPS> trace_ray_; // Warp-uniform half of the ray, staged by the config uop: it rides the arm // doorbell in HW, so it is not a window write either. - std::array, + std::array, VX_CFG_NUM_WARPS> trace_scene_; - std::array trace_payload_{}; + std::array trace_payload_{}; std::array trace_flags_{}; std::array trace_cull_{}; // The slot handle each lane's outstanding candidate came from, mirrored here diff --git a/sim/simx/types.h b/sim/simx/types.h index 6423d1a61a..1658771c6e 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -895,7 +895,7 @@ enum class ArbiterType { Priority, RoundRobin, Matrix, - GTO + GreedyRR }; inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { @@ -903,7 +903,7 @@ inline std::ostream &operator<<(std::ostream &os, const ArbiterType& type) { case ArbiterType::Priority: os << "Priority"; break; case ArbiterType::RoundRobin: os << "RoundRobin"; break; case ArbiterType::Matrix: os << "Matrix"; break; - case ArbiterType::GTO: os << "GTO"; break; + case ArbiterType::GreedyRR: os << "GreedyRR"; break; default: assert(false); } return os; @@ -1017,56 +1017,36 @@ class MatrixArbiter : public IArbiterImpl { std::vector> priority_matrix_; }; -class GTOArbiter : public IArbiterImpl { +// Greedy round-robin: keeps granting the current owner while it requests, +// then hands over to the next requester after it in index order. +class GreedyRRArbiter : public IArbiterImpl { public: - GTOArbiter(uint32_t size) - : size_(size) - , age_(size, 0) - , last_grant_(-1u) { + GreedyRRArbiter(uint32_t size) : size_(size) { this->reset(); } uint32_t grant(const BitVector<>& requests) override { assert(requests.size() == size_); - // greedy: keep granting same requester if still active if (last_grant_ < size_ && requests.test(last_grant_)) { - this->update_ages(requests, last_grant_); return last_grant_; } - // Then-Oldest: find the requester with the highest age - uint32_t best = -1u; - uint32_t best_age = 0; + uint32_t start = (last_grant_ < size_) ? (last_grant_ + 1) % size_ : 0; for (uint32_t i = 0; i < size_; ++i) { - if (requests.test(i) && (best == -1u || age_[i] > best_age)) { - best = i; - best_age = age_[i]; + uint32_t idx = (start + i) % size_; + if (requests.test(idx)) { + last_grant_ = idx; + return idx; } } - if (best != -1u) { - last_grant_ = best; - this->update_ages(requests, best); - } - return best; + return -1u; } void reset() override { - std::fill(age_.begin(), age_.end(), 0); last_grant_ = -1u; } private: - void update_ages(const BitVector<>& requests, uint32_t granted) { - for (uint32_t i = 0; i < size_; ++i) { - if (i == granted || !requests.test(i)) { - age_[i] = 0; - } else { - ++age_[i]; - } - } - } - uint32_t size_; - std::vector age_; uint32_t last_grant_; }; @@ -1083,8 +1063,8 @@ class Arbiter { case ArbiterType::Matrix: impl_ = std::make_shared(size); break; - case ArbiterType::GTO: - impl_ = std::make_shared(size); + case ArbiterType::GreedyRR: + impl_ = std::make_shared(size); break; default: assert(false); diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index b2e9be960a..4e51883d19 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -7,8 +7,8 @@ SRC_DIR := $(VORTEX_HOME)/sim/xrtsim AFU_DIR := $(RTL_DIR)/afu/xrt AFU_COMMON_DIR := $(RTL_DIR)/afu/common -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Wno-array-bounds -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -fPIC # GCC 13 turns the generated per-word copy of a wide signal into a memcpy and # inlines it with aligned 16-byte stores, faulting on a destination that is only # 8-byte aligned in the root class (NUM_THREADS=16 + RTU crashes in reset). @@ -22,7 +22,7 @@ CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) LDFLAGS += -shared $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a -LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -pthread +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator -L$(THIRD_PARTY_DIR)/ramulator -lramulator -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state -pthread # control RTL debug tracing states DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE @@ -141,7 +141,7 @@ RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic -VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO -Wno-fatal +VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) @@ -216,8 +216,10 @@ $(DESTDIR)/vortex.json: $(CONFIG_FILE) $(DESTDIR)/scope.json: $(DESTDIR)/vortex.json $(SCRIPT_DIR)/scope.py $^ -o $@ +# verilator --build runs its own make (parallelized by -j in VL_FLAGS) that +# cannot reach this make's jobserver, so it gets a clean MAKEFLAGS. $(DESTDIR)/$(PROJECT): $(SRCS) $(SCOPE_JSON) $(CONFIG_FILE) $(RTL_SRCS) - $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + MAKEFLAGS= $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(TOP) --top-module $(TOP) $(SRCS) -CFLAGS '$(CXXFLAGS)' -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ # updates the timestamp when flags changed. $(CONFIG_FILE): force diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index fcab150cfe..af948d92dc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -50,9 +50,6 @@ #include -#ifndef MEM_CLOCK_RATIO -#define MEM_CLOCK_RATIO 1 -#endif #define CACHE_BLOCK_SIZE 64 @@ -144,7 +141,7 @@ class xrt_sim::Impl { , ram_{} , host_ram_(nullptr) , host_alloc_(nullptr) - , dram_sim_(VX_CFG_PLATFORM_MEMORY_NUM_BANKS, VX_CFG_PLATFORM_MEMORY_DATA_SIZE, MEM_CLOCK_RATIO) + , dram_sim_(DramSim::platform_config()) , stop_(false) #ifdef VCD_OUTPUT , tfp_(nullptr) @@ -536,9 +533,9 @@ class xrt_sim::Impl { dram_sim_.tick(); for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty()) { + if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(mem_req->addr, mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -556,6 +553,19 @@ class xrt_sim::Impl { #endif } + // Byte address within its bank (DRAM channel) of an AXI bank address: + // VX_mem_to_axi places the bank select above the data-size bits when the + // banks are interleaved. + static uint64_t bank_local_addr(uint64_t axi_addr) { + #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 + constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; + #else + return axi_addr; + #endif + } + void eval() { device_->eval(); #ifdef VCD_OUTPUT diff --git a/sw/common/gfx_setup.h b/sw/common/gfx_setup.h index 3698cd84f7..7b9d619d00 100644 --- a/sw/common/gfx_setup.h +++ b/sw/common/gfx_setup.h @@ -155,6 +155,24 @@ static inline bool EdgeEquation(vec3f edges[3], const vec4f& v0, return true; } +// A depth-plane term in Q7.24, stored modulo 2^32. The constant term is the plane +// at the screen origin, which leaves Q7.24's +-128 range for a steep plane even +// though the depth at every covered pixel is in range, and an out-of-range float +// to int conversion is undefined (RISC-V saturates, x86 yields INT_MIN). Late-Z +// and early-Z both evaluate the plane mod 2^32, so the wrapped term is exact. A +// term past 2^39 (non-finite, or a degenerate sliver) has no usable depth. +static inline FloatA PlaneToFixed(float f) { + float scaled = f * 16777216.0f; // 2^24 + float mag = __builtin_fabsf(f); + if (mag < 128.0f) { + return FloatA::make((int32_t)scaled); + } + if (!(mag < 549755813888.0f)) { // 2^39: scaled would overflow int64 + return FloatA::make(0); + } + return FloatA::make((int32_t)(uint32_t)(uint64_t)(int64_t)scaled); +} + // Normalize the edge matrix by 1/maxVal and convert to Q15.16. static inline void EdgeToFixed(vec3e_t out[3], const vec3f in[3]) { float maxVal = __builtin_fabsf(in[0].x); @@ -271,9 +289,9 @@ static inline bool setup_triangle(const setup_vertex_t& v0, // C' evaluated so an integer-pixel MAC samples Z at the pixel center // (X+0.5, Y+0.5) — matching the +0.5 half-pixel offset the coverage edges use. float zc = ps0.z - za * ps0.x - zb * ps0.y + 0.5f * (za + zb); - out_prim.attribs.z.x = FloatA(za); - out_prim.attribs.z.y = FloatA(zb); - out_prim.attribs.z.z = FloatA(zc); + out_prim.attribs.z.x = PlaneToFixed(za); + out_prim.attribs.z.y = PlaneToFixed(zb); + out_prim.attribs.z.z = PlaneToFixed(zc); } // Perspective-correct varyings: premultiply each colour/texcoord by the // vertex's 1/w and carry a separate 1/w plane; the FS interpolates every plane diff --git a/sw/common/gfx_sw.h b/sw/common/gfx_sw.h index 503b9721f9..96405a48d8 100644 --- a/sw/common/gfx_sw.h +++ b/sw/common/gfx_sw.h @@ -399,11 +399,11 @@ static inline __attribute__((always_inline)) bool om_sample_rmw( // stride gather. static inline __attribute__((always_inline)) uintptr_t msaa_color_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)(x * samples + sample) * om_color_bpp(s.color_format)); + return (uintptr_t)(s.cbuf_base + (uint64_t)y * s.cbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_color_bpp(s.color_format)); } static inline __attribute__((always_inline)) uintptr_t msaa_depth_addr( const om_state_t& s, uint32_t samples, uint32_t x, uint32_t y, uint32_t sample) { - return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)(x * samples + sample) * om_depth_bpp(s.depth_format)); + return (uintptr_t)(s.zbuf_base + (uint64_t)y * s.zbuf_pitch + (uint64_t)((uintptr_t)x * samples + sample) * om_depth_bpp(s.depth_format)); } // Box resolve of one pixel's S color samples → a single ARGB8888 value diff --git a/sw/common/linked_list.h b/sw/common/linked_list.h index f75a6b8dae..38f175ed54 100644 --- a/sw/common/linked_list.h +++ b/sw/common/linked_list.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include namespace vortex { @@ -238,7 +239,7 @@ class LinkedList { } void push_back(T *obj) { - insert(iterator(tail_), obj); + insert(end(), obj); } void push_front(T *obj) { diff --git a/sw/gfx/gfx_frontend_k.h b/sw/gfx/gfx_frontend_k.h index 9e557ab7e3..e2ee763c97 100644 --- a/sw/gfx/gfx_frontend_k.h +++ b/sw/gfx/gfx_frontend_k.h @@ -85,8 +85,8 @@ __kernel void expand_k(expand_arg_t* __UNIFORM__ arg) { const uint32_t nv = arg->num_varyings; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t i = gid; i < n; i += gstride) { - const uint8_t* rec = recs + (size_t)i * stride; + for (size_t i = gid; i < n; i += gstride) { + const uint8_t* rec = recs + i * stride; const float* pos = reinterpret_cast(rec); setup_vertex_t v; v.pos[0] = pos[0]; v.pos[1] = pos[1]; v.pos[2] = pos[2]; v.pos[3] = pos[3]; @@ -156,7 +156,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // traffic and removes the slot_prim/slot_bbox scratch entirely. uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; keep[t] = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -183,7 +183,7 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { // here equals keep[t] from SETUP (same pure function, same inputs). uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = pipe_clip_and_setup(&verts[3 * t], W, H, arg->cull_mode, vpp, pr, bb); @@ -210,12 +210,12 @@ __kernel void setup_k(pipe_arg_t* __UNIFORM__ arg) { for (uint32_t c = 0; c < 4; ++c) { fw[2 + c] = pv[4 + c]; } for (uint32_t c = 0; c < 6; ++c) { fw[6 + c] = pv[10 + c]; } } - uint32_t w = offset[t]; + size_t w = offset[t]; for (uint32_t s = 0; s < kept; ++s) { prim[w] = pr[s]; bbox[w] = bb[s]; if (flat) { - uint32_t* dst = flat + (size_t)w * GFX_FS_FLAT_WORDS; + uint32_t* dst = flat + w * GFX_FS_FLAT_WORDS; for (uint32_t c = 0; c < GFX_FS_FLAT_WORDS; ++c) { dst[c] = fw[c]; } } ++w; @@ -250,7 +250,7 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -275,9 +275,9 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; pipe_bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * arg->bin_cols + bx) << PIPE_PRIM_BITS) | i; @@ -289,11 +289,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { uint32_t K = meta[1]; uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo; b < hi; ++b) thist[tid * B + b] = 0; + for (uint32_t b = lo; b < hi; ++b) thist[(size_t)tid * B + b] = 0; __syncthreads(); - for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } + for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[(size_t)tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[(size_t)t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { @@ -341,11 +341,11 @@ __kernel void binning_k(pipe_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + B * sizeof(rast_bin_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = pipe_umin(tid * kchunk, K), khi = pipe_umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[(size_t)t * B + b]; thist[(size_t)t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; - if (b >= lo && b < hi) pids[thist[tid * B + b]++] = key & PIPE_PRIM_MASK; + if (b >= lo && b < hi) pids[thist[(size_t)tid * B + b]++] = key & PIPE_PRIM_MASK; } } break; diff --git a/sw/kernel/Makefile b/sw/kernel/Makefile index a00ea753a8..668b6f8c18 100644 --- a/sw/kernel/Makefile +++ b/sw/kernel/Makefile @@ -29,6 +29,7 @@ DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy CFLAGS += -O3 -mcmodel=medany -fno-exceptions -fdata-sections -ffunction-sections -D__VORTEX__ +CFLAGS += -Wall -Wextra -Wfatal-errors -Werror CFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') CFLAGS += $(XCONFIGS) diff --git a/sw/kernel/include/vx_dxa.h b/sw/kernel/include/vx_dxa.h index ea722b2752..527cd89da2 100644 --- a/sw/kernel/include/vx_dxa.h +++ b/sw/kernel/include/vx_dxa.h @@ -49,15 +49,15 @@ extern "C" { // mask, so it reads them from the warp-gather registers the gathers wrote // (x31, and x30 for rs2; see __VX_WGATHER_IN). #define __VX_DXA_ISSUE1(a0v) do { \ - register uint32_t __rs1 __asm__("x31") = (a0v); \ + register size_t __rs1 __asm__("x31") = (a0v); \ __asm__ volatile (".insn r %0, 0, %1, x0, %2, x0\n\t" \ : : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(__rs1) \ : "memory"); \ } while (0) #define __VX_DXA_ISSUE2(a0v, a1v) do { \ - register uint32_t __rs1 __asm__("x31") = (a0v); \ - register uint32_t __rs2 __asm__("x30") = (a1v); \ + register size_t __rs1 __asm__("x31") = (a0v); \ + register size_t __rs2 __asm__("x30") = (a1v); \ __asm__ volatile (".insn r %0, 0, %1, x0, %2, %3\n\t" \ : : "i"(VX_DXA_EXT_OPCODE), "i"(VX_DXA_FUNCT7), "r"(__rs1), \ "r"(__rs2) \ @@ -74,10 +74,10 @@ inline void vx_dxa_issue_1d_wg(uint32_t desc_slot, const void* smem_addr, uint32_t coord0) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); __VX_DXA_ISSUE1(a0); } @@ -88,10 +88,10 @@ inline void vx_dxa_issue_2d_wg(uint32_t desc_slot, uint32_t coord0, uint32_t coord1) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); __VX_DXA_ISSUE1(a0); } @@ -103,14 +103,14 @@ inline void vx_dxa_issue_3d_wg(uint32_t desc_slot, uint32_t coord1, uint32_t coord2) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)0u, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)0u, + (size_t)0u, + (size_t)0u); __VX_DXA_ISSUE2(a0, a1); } @@ -122,14 +122,14 @@ inline void vx_dxa_issue_4d_wg(uint32_t desc_slot, uint32_t coord2, uint32_t coord3) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)coord3, - (size_t)0u, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)coord3, + (size_t)0u, + (size_t)0u); __VX_DXA_ISSUE2(a0, a1); } @@ -142,14 +142,14 @@ inline void vx_dxa_issue_5d_wg(uint32_t desc_slot, uint32_t coord3, uint32_t coord4) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)0u); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)0u); __VX_DXA_ISSUE2(a0, a1); } @@ -163,14 +163,14 @@ inline void vx_dxa_issue_1d_multicast_wg(uint32_t desc_slot, uint32_t coord0, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)0u); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)0u); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __VX_DXA_ISSUE2(a0, a1); } @@ -182,14 +182,14 @@ inline void vx_dxa_issue_2d_multicast_wg(uint32_t desc_slot, uint32_t coord1, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)0, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)0, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __VX_DXA_ISSUE2(a0, a1); } @@ -202,14 +202,14 @@ inline void vx_dxa_issue_3d_multicast_wg(uint32_t desc_slot, uint32_t coord2, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)0, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)0, + (size_t)0, + (size_t)cta_mask); __VX_DXA_ISSUE2(a0, a1); } @@ -223,14 +223,14 @@ inline void vx_dxa_issue_4d_multicast_wg(uint32_t desc_slot, uint32_t coord3, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)coord3, - (size_t)0, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)coord3, + (size_t)0, + (size_t)cta_mask); __VX_DXA_ISSUE2(a0, a1); } @@ -245,14 +245,14 @@ inline void vx_dxa_issue_5d_multicast_wg(uint32_t desc_slot, uint32_t coord4, uint32_t cta_mask) { const uint32_t meta = vx_dxa_pack_meta(desc_slot, barrier_id); - const uint32_t a0 = (uint32_t)vx_wgather((size_t)(uintptr_t)smem_addr, - (size_t)meta, - (size_t)coord0, - (size_t)coord1); - const uint32_t a1 = (uint32_t)__VX_WGATHER_IN("x30", 0, (size_t)coord2, - (size_t)coord3, - (size_t)coord4, - (size_t)cta_mask); + const size_t a0 = vx_wgather((size_t)(uintptr_t)smem_addr, + (size_t)meta, + (size_t)coord0, + (size_t)coord1); + const size_t a1 = __VX_WGATHER_IN("x30", 0, (size_t)coord2, + (size_t)coord3, + (size_t)coord4, + (size_t)cta_mask); __VX_DXA_ISSUE2(a0, a1); } diff --git a/sw/kernel/include/vx_graphics.h b/sw/kernel/include/vx_graphics.h index 0eb7b094a6..67664edf5b 100644 --- a/sw/kernel/include/vx_graphics.h +++ b/sw/kernel/include/vx_graphics.h @@ -68,8 +68,9 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // XBITS/YBITS/RECORD_SHIFT come from the OM DCRs; the runtime programs them and // passes them to the kernel, so the shader just shifts and adds. #define VX_OM_APERTURE_ADDR_RT(xbits, ybits, record_shift, x, y, face, rt) \ - ((VX_MEM_OM_BASE_ADDR) + \ - (((((uint32_t)(rt) << 1 | (uint32_t)(face)) << ((xbits) + (ybits))) \ + ((uintptr_t)(VX_MEM_OM_BASE_ADDR) + \ + (uintptr_t)(((((uint32_t)(rt) << 1 | (uint32_t)(face)) \ + << ((xbits) + (ybits))) \ | ((uint32_t)(y) << (xbits)) \ | (uint32_t)(x)) << (record_shift))) @@ -84,7 +85,7 @@ inline unsigned vx_tex(unsigned stage, unsigned u, unsigned v, unsigned lod) { // into one ordinary store per word; the LSU never learns that OM exists. #define vx_om_export(addr, color, depth, mask) \ __asm__ volatile (".insn r4 %0, 3, %1, x0, %2, %3, %4" \ - :: "i"(RISCV_CUSTOM1), "i"(mask), "r"(addr), "r"(color), "r"(depth)) + :: "i"(RISCV_CUSTOM1), "i"(mask), "r"((uintptr_t)(addr)), "r"(color), "r"(depth)) // The three record shapes. #define vx_om_export_color(addr, color) vx_om_export(addr, color, 0, 1) diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 427169096e..78468f7744 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -614,7 +614,7 @@ inline void vx_barrier_expect_tx(int barrier_id, int count) { // | (MEM8[base + 2 * stride] << 16) // | (MEM8[base + 3 * stride] << 24)); __attribute__((always_inline)) -inline float vx_packlb_f(const void* base, uint32_t stride) { +inline float vx_packlb_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 1, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" @@ -628,7 +628,7 @@ inline float vx_packlb_f(const void* base, uint32_t stride) { // fd = bitcast((MEM16[base + 0 * stride] << 0) // | (MEM16[base + 1 * stride] << 16)); __attribute__((always_inline)) -inline float vx_packlh_f(const void* base, uint32_t stride) { +inline float vx_packlh_f(const void* base, size_t stride) { float out; __asm__ volatile ( ".insn r %1, 2, 4, %0, %2, %3" : "=f"(out) : "i"(RISCV_CUSTOM0), "r"(base), "r"(stride) : "memory" diff --git a/sw/kernel/include/vx_print.h b/sw/kernel/include/vx_print.h index 696d297759..6df557009f 100644 --- a/sw/kernel/include/vx_print.h +++ b/sw/kernel/include/vx_print.h @@ -20,8 +20,8 @@ extern "C" { #endif -int vx_vprintf(const char* format, va_list va); -int vx_printf(const char * format, ...); +int vx_vprintf(const char* format, va_list va) __attribute__((format(printf, 1, 0))); +int vx_printf(const char * format, ...) __attribute__((format(printf, 1, 2))); void vx_putchar(int c); void vx_putint(int value, int base); diff --git a/sw/kernel/include/vx_raytrace.h b/sw/kernel/include/vx_raytrace.h index de4d87ec47..fe092afd1a 100644 --- a/sw/kernel/include/vx_raytrace.h +++ b/sw/kernel/include/vx_raytrace.h @@ -50,6 +50,17 @@ extern "C" { : "memory"); \ __v; }) +// vx_rt_get_payload — the tracing shader's payload pointer, as the RTU staged it +// for this record (VX_RT_PAYLOAD_PTR_LO, plus the next slot on RV64). +static inline __attribute__((always_inline)) +uintptr_t vx_rt_get_payload(uint32_t tok) { + uintptr_t ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, tok); +#if __riscv_xlen == 64 + ptr |= (uintptr_t)vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO + 1, tok) << 32; +#endif + return ptr; +} + // vx_rt_cb_ret — release the lane's parked context in the RTU with one of // VX_RT_CB_{ACCEPT,IGNORE,TERMINATE}, handing back the intersection shader's // computed hit distance and attribute in the same instruction. The callback @@ -142,13 +153,13 @@ _Static_assert(offsetof(vx_hit_t, instance_custom) == 6 * sizeof(float), "vx_hit // across the warp's lanes, and the f0..f7 ray window is per-lane. // scene_ptr / payload_ptr / ray_flags / cull_mask : per-trace warp-uniform // config, lane-packed into one register via an IMPLICIT vx_wgather -// (lane0=scene, lane1=payload, lane2=flags, lane3=cull) — pure register +// (lane1=scene, lane2=payload, lane3=flags|cull) — pure register // domain, no memory traffic; hoists out of a bounce loop when invariant. // ray : the per-thread geometry, pinned into the f0..f7 caller-saved window. // Returns the async ray handle in rd. Non-blocking: the RTU traverses // while the kernel runs independent work; vx_rt_wait is the sync point. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray) { // Pack config into the GATHERED operands (not the wgather self slot): the @@ -157,7 +168,7 @@ uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, // lane3={flags,cull}; lane0 (self) unused. Keeps scene valid even when lane 0 // is masked (callback/recursion-narrowed traces). uint32_t flags_cull = (ray_flags & 0xffffu) | (cull_mask << 16); - uint32_t cfg = (uint32_t)vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); + size_t cfg = vx_wgather(0u, scene_ptr, payload_ptr, flags_cull); register float r0 __asm__("f0") = ray->origin[0]; register float r1 __asm__("f1") = ray->origin[1]; register float r2 __asm__("f2") = ray->origin[2]; @@ -173,7 +184,7 @@ uint32_t vx_rt_wtrace(uint32_t scene_ptr, uint32_t payload_ptr, // the field references stable across the long register-binding list. // The RTU reads the packed config lanes regardless of the thread mask, so it // reads them from the warp-gather register itself (see __VX_WGATHER_IN). - register uint32_t cfg_reg __asm__("x31") = cfg; + register size_t cfg_reg __asm__("x31") = cfg; __asm__ volatile (".insn r %[op], 7, 0, %[hnd], %[cfg], x0" : [hnd]"=r"(handle) : [op]"i"(RISCV_CUSTOM1), [cfg]"r"(cfg_reg), @@ -333,7 +344,7 @@ void vx_rt_get_objray(vx_objray_t* out) { // have independent work, it calls trace/wait separately and the compiler // schedules that work into the gap. static inline __attribute__((always_inline)) -uint32_t vx_rt_wtrace_sync(uint32_t scene_ptr, uint32_t payload_ptr, +uint32_t vx_rt_wtrace_sync(uintptr_t scene_ptr, uintptr_t payload_ptr, uint32_t ray_flags, uint32_t cull_mask, const vx_ray_t* ray, vx_hit_t* hit) { uint32_t h = vx_rt_wtrace(scene_ptr, payload_ptr, ray_flags, cull_mask, ray); diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 4106404b6b..618a95b030 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -87,7 +87,7 @@ namespace detail { } } - static __attribute__((always_inline)) D pack_row(const Type *base, uint32_t ldm) { + static __attribute__((always_inline)) D pack_row(const Type *base, size_t ldm) { static_assert(sizeof(D) % sizeof(Type) == 0, "D must be a multiple of Type in size"); if constexpr (sizeof(Type) == 1 && sizeof(D) == 4) { // 4 × 1-byte strided loads → single pack-load byte instruction @@ -368,7 +368,7 @@ struct wmma_context { k_bases[0] = base; if constexpr (num_k_groups >= 2) { asm volatile("" : "+r"(k_bases[0])); // prevent reverse strength reduction - auto k_ldm_step = k_stride * (uint32_t)ldm; + size_t k_ldm_step = k_stride * ldm; detail::unroll_for([&](auto i) { k_bases[i + 1] = k_bases[i] + k_ldm_step; }); @@ -393,7 +393,7 @@ struct wmma_context { n_bases[0] = base; if constexpr (num_n_groups >= 2) { asm volatile("" : "+r"(n_bases[0])); // prevent reverse strength reduction - auto n_ldm_step = n_stride * (uint32_t)ldm; + size_t n_ldm_step = n_stride * ldm; detail::unroll_for([&](auto i) { n_bases[i + 1] = n_bases[i] + n_ldm_step; }); @@ -864,7 +864,7 @@ struct wgmma_context { uint32_t k_in_elem = (lane % tcK) * i_ratio; auto load_reg = [&](uint32_t m_blk, uint32_t k_blk) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; @@ -873,7 +873,7 @@ struct wgmma_context { + k_in_elem; } else { // Row-major SMEM: standard row*ldm + col. - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } return *reinterpret_cast( @@ -893,14 +893,14 @@ struct wgmma_context { uint32_t m_blk = r / a_reg_k_steps; uint32_t k_blk = r % a_reg_k_steps; if constexpr (k_steps == 1) { - uint32_t elem_off; + size_t elem_off; if (ldm == 0) { elem_off = m_blk * a_blk_elems + i_in * (fedpK * i_ratio) + k_blk * k_row_elems + k_in_elem; } else { - elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + elem_off = (m_blk * tcM + i_in) * ldm + (k_blk * k_row_elems + k_in_elem); } dst.data[r] = *reinterpret_cast( diff --git a/sw/kernel/src/common.h b/sw/kernel/src/common.h index af05d63fe2..6256141855 100644 --- a/sw/kernel/src/common.h +++ b/sw/kernel/src/common.h @@ -13,7 +13,7 @@ #define RISCV_CUSTOM0 0x0B -#ifdef VX_CFG_XLEN_64 +#if __riscv_xlen == 64 #define LOAD_IMMEDIATE64(rd, imm) \ li t0, (imm >> 32); \ slli t0, t0, 32; \ diff --git a/sw/kernel/src/vx_print.c b/sw/kernel/src/vx_print.c index e23f0719ad..1056353031 100644 --- a/sw/kernel/src/vx_print.c +++ b/sw/kernel/src/vx_print.c @@ -43,7 +43,7 @@ typedef struct { static void __putint_cb(const putint_arg_t* arg) { char tmp[33]; - float value = arg->value; + int value = arg->value; int base = arg->base; itoa(value, tmp, base); for (int i = 0; i < 33; ++i) { diff --git a/sw/kernel/src/vx_serial.S b/sw/kernel/src/vx_serial.S index 6d6d8ba827..0cea0704a7 100644 --- a/sw/kernel/src/vx_serial.S +++ b/sw/kernel/src/vx_serial.S @@ -18,8 +18,8 @@ .type vx_serial, @function .global vx_serial vx_serial: -#if (VX_CFG_XLEN == 64) - addi sp, sp, -56 +#if __riscv_xlen == 64 + addi sp, sp, -64 sd ra, 48(sp) sd s5, 40(sp) sd s4, 32(sp) @@ -28,7 +28,7 @@ vx_serial: sd s1, 8(sp) sd s0, 0(sp) #else - addi sp, sp, -28 + addi sp, sp, -32 sw ra, 24(sp) sw s5, 20(sp) sw s4, 16(sp) @@ -53,7 +53,7 @@ label_join: .insn r RISCV_CUSTOM0, 3, 0, x0, s5, x0 # join s5 addi s0, s0, 1 # index++ blt s0, s2, label_loop # loop back -#if (VX_CFG_XLEN == 64) +#if __riscv_xlen == 64 ld ra, 48(sp) ld s5, 40(sp) ld s4, 32(sp) @@ -61,7 +61,7 @@ label_join: ld s2, 16(sp) ld s1, 8(sp) ld s0, 0(sp) - addi sp, sp, 56 + addi sp, sp, 64 #else lw ra, 24(sp) lw s5, 20(sp) @@ -70,6 +70,6 @@ label_join: lw s2, 8(sp) lw s1, 4(sp) lw s0, 0(sp) - addi sp, sp, 28 + addi sp, sp, 32 #endif ret diff --git a/sw/kernel/src/vx_spawn.c b/sw/kernel/src/vx_spawn.c index af8d9c2c65..fecb6fdfb7 100644 --- a/sw/kernel/src/vx_spawn.c +++ b/sw/kernel/src/vx_spawn.c @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -144,7 +145,7 @@ static void __attribute__ ((noinline)) process_thread_groups() { for (uint32_t group_id = start_group; group_id < end_group; group_id += group_stride) { blockIdx.x = group_id % gridDim_x; blockIdx.y = (group_id / gridDim_x) % gridDim_y; - blockIdx.z = group_id / (gridDim_x * gridDim_y); + blockIdx.z = group_id / gridDim_xy; callback((void*)arg); } } @@ -155,7 +156,7 @@ static void __attribute__ ((noinline)) process_thread_groups_stub() { uint32_t remaining_mask = targs->remaining_mask; uint32_t warp_id = vx_warp_id(); uint32_t group_warp_id = warp_id % warps_per_group; - uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : -1; + uint32_t threads_mask = (group_warp_id == warps_per_group-1) ? remaining_mask : ~0u; // activate threads vx_tmc(threads_mask); @@ -192,7 +193,7 @@ int vx_spawn_threads(uint32_t dimension, // check group size uint32_t threads_per_core = warps_per_core * threads_per_warp; if (threads_per_core < group_size) { - vx_printf("error: group_size > threads_per_core (%d,%d)\n", group_size, threads_per_core); + vx_printf("error: group_size > threads_per_core (%" PRIu32 ",%" PRIu32 ")\n", group_size, threads_per_core); return -1; } diff --git a/sw/kernel/src/vx_start.S b/sw/kernel/src/vx_start.S index 60d47d8125..5467253caa 100644 --- a/sw/kernel/src/vx_start.S +++ b/sw/kernel/src/vx_start.S @@ -261,5 +261,11 @@ init_tls_all: .section .data .weak __dso_handle +#if __riscv_xlen == 64 + .p2align 3 +__dso_handle: + .dword 0 +#else __dso_handle: .long 0 +#endif diff --git a/sw/kernel/src/vx_syscalls.c b/sw/kernel/src/vx_syscalls.c index b96ff6f17f..4f34771e32 100644 --- a/sw/kernel/src/vx_syscalls.c +++ b/sw/kernel/src/vx_syscalls.c @@ -17,37 +17,69 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { #endif -int _close(int file) { return -1; } +int _close(int file) { + (void)file; + return -1; +} -int _fstat(int file, struct stat *st) { return -1; } +int _fstat(int file, struct stat *st) { + (void)file; + (void)st; + return -1; +} -int _isatty(int file) { return 0; } +int _isatty(int file) { + (void)file; + return 0; +} -int _lseek(int file, int ptr, int dir) { return 0; } +off_t _lseek(int file, off_t ptr, int dir) { + (void)file; + (void)ptr; + (void)dir; + return 0; +} -int _open(const char *name, int flags, int mode) { return -1; } +int _open(const char *name, int flags, int mode) { + (void)name; + (void)flags; + (void)mode; + return -1; +} -int _read(int file, char *ptr, int len) { return -1; } +ssize_t _read(int file, void *ptr, size_t len) { + (void)file; + (void)ptr; + (void)len; + return -1; +} -caddr_t _sbrk(int incr) { +void *_sbrk(ptrdiff_t incr) { + (void)incr; __asm__ __volatile__("ebreak"); return 0; } -int _write(int file, char *ptr, int len) { - int i; - for (i = 0; i < len; ++i) { - vx_putchar(*ptr++); +ssize_t _write(int file, const void *ptr, size_t len) { + (void)file; + const char *p = (const char *)ptr; + for (size_t i = 0; i < len; ++i) { + vx_putchar(*p++); } return len; } -int _kill(int pid, int sig) { return -1; } +int _kill(int pid, int sig) { + (void)pid; + (void)sig; + return -1; +} int _getpid() { return vx_hart_id(); diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ca30664017..ddb1def115 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -37,6 +37,14 @@ static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; +// Warps, sockets per cluster and clusters carry an exact count, so a machine +// that is not a power of two reports truly; the field is zero on devices +// built before it existed, which report only the 3-bit log2 field. +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; + return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, @@ -51,20 +59,20 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = 1ull << ((dev_caps >> 8) & 0x7); return true; case VX_CAPS_NUM_WARPS: - *out = 1ull << ((dev_caps >> 11) & 0x7); + *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint32_t socket_size = 1u << ((dev_caps >> 14) & 0x7); - uint32_t cluster_size = 1u << ((dev_caps >> 17) & 0x7); - uint32_t num_clusters = 1u << ((dev_caps >> 20) & 0x7); - *out = uint64_t(num_clusters) * cluster_size * socket_size; + uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); + uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); + uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); + *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: *out = 1ull << ((dev_caps >> 14) & 0x7); return true; case VX_CAPS_NUM_CLUSTERS: - *out = 1ull << ((dev_caps >> 20) & 0x7); + *out = exact_or_log2(dev_caps, 57, 0x7F, 20); return true; case VX_CAPS_ISSUE_WIDTH: *out = 1ull << ((dev_caps >> 23) & 0x7); diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 3c539fd521..0922d8cfd1 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,12 @@ VMManager::VMManager(DeviceMemIO* dev_io) std::cout << "[VM] Virtual address randomization ENABLED (seed=0x" << std::hex << seed << std::dec << ")" << std::endl; } + + const char* megapage_env = std::getenv("VORTEX_VM_MEGAPAGE_HEAP"); + megapage_heap_ = (megapage_env != nullptr && std::atoi(megapage_env) != 0); + if (megapage_heap_) { + std::cout << "[VM] Heap mapped with superpages" << std::endl; + } } VMManager::~VMManager() { @@ -114,6 +121,22 @@ int VMManager::init() { GLOBAL_MEM_SIZE - VX_MEM_PAGE_TABLE_BASE_ADDR), { return err; }); + if (megapage_heap_) { + // Superpages over the whole heap, as GPU drivers back allocations with + // 2 MB pages: a buffer keeps its identity VA and no longer costs a TLB + // entry per 4 KB. Per-buffer access flags are not enforced in this mode. + if (randomize_va_) { + std::cerr << "[VM] VORTEX_VM_MEGAPAGE_HEAP and VORTEX_RANDOMIZE_VA are mutually exclusive" << std::endl; + return -1; + } + uint64_t heap_end = VX_MEM_PAGE_TABLE_BASE_ADDR; +#if VX_VM_ADDR_MODE == SV32 + heap_end = std::min(heap_end, 0x100000000ULL); +#endif + CHECK_ERR(install_identity_map(ALLOC_BASE_ADDR, heap_end - ALLOC_BASE_ADDR), { + return err; + }); + } } // install_identity_map() already flushed; an extra flush here is a // cheap no-op if nothing else dirtied the shadow. @@ -183,6 +206,9 @@ uint64_t VMManager::map_p2v(uint64_t ppn, uint32_t flags) { int VMManager::phy_to_virt_map(uint64_t size, uint64_t* dev_pAddr, uint32_t flags) { if (!need_trans(*dev_pAddr)) return 0; + if (megapage_heap_ && *dev_pAddr >= ALLOC_BASE_ADDR && *dev_pAddr < VX_MEM_PAGE_TABLE_BASE_ADDR) { + return 0; // identity-mapped by init() + } uint64_t init_pAddr = *dev_pAddr; // Round up: a sub-page allocation still needs one PTE. A plain diff --git a/sw/runtime/common/vm.h b/sw/runtime/common/vm.h index e922a580aa..581ecd43fe 100644 --- a/sw/runtime/common/vm.h +++ b/sw/runtime/common/vm.h @@ -126,6 +126,9 @@ class VMManager { // Randomized VA allocation (VORTEX_RANDOMIZE_VA / VORTEX_VA_SEED). std::mt19937_64 rng_; bool randomize_va_; + + // Heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP). + bool megapage_heap_; }; } // namespace vortex diff --git a/sw/runtime/simx/Makefile b/sw/runtime/simx/Makefile index 65ec2c4c7a..39c40cc70f 100644 --- a/sw/runtime/simx/Makefile +++ b/sw/runtime/simx/Makefile @@ -6,7 +6,7 @@ CONFIG_FILE := $(DESTDIR)/simx_rt_config.stamp SRC_DIR := $(VORTEX_HOME)/sw/runtime/simx CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror -CXXFLAGS += -fPIC -Wno-maybe-uninitialized +CXXFLAGS += -fPIC CXXFLAGS += -I$(INC_DIR) -I../common -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_DIR)/simx -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) CXXFLAGS += $(CONFIGS) diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 0b315896d2..77c80baa6e 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -83,7 +83,7 @@ POCL_CC_FLAGS += POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_CC_FLAGS += POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_CC_FLAGS += POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" -HIPCC_FLAGS += -std=c++17 +HIPCC_FLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror HIPCC_FLAGS += $(CONFIGS) # The embedded .hipInfo OFFLOAD_TRIPLE selects the primary SPIR-V pointer # width. hipcc rewrites it to match XLEN so the emitted SPIR-V is diff --git a/tests/hip/sgemm/main.cpp b/tests/hip/sgemm/main.cpp index 26de4d7233..99b63c0da8 100644 --- a/tests/hip/sgemm/main.cpp +++ b/tests/hip/sgemm/main.cpp @@ -89,8 +89,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size_sq; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/hip/vecadd/main.cpp b/tests/hip/vecadd/main.cpp index 9532b98eec..5e8e8b7e4b 100644 --- a/tests/hip/vecadd/main.cpp +++ b/tests/hip/vecadd/main.cpp @@ -71,8 +71,8 @@ int main(int argc, char** argv) { srand(50); for (uint32_t i = 0; i < size; ++i) { - h_a[i] = static_cast(rand()) / RAND_MAX; - h_b[i] = static_cast(rand()) / RAND_MAX; + h_a[i] = static_cast(rand()) / static_cast(RAND_MAX); + h_b[i] = static_cast(rand()) / static_cast(RAND_MAX); } printf("Allocate device buffers\n"); diff --git a/tests/mpi/mpi_blocked_sgemm/kernel.cpp b/tests/mpi/mpi_blocked_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_blocked_sgemm/kernel.cpp +++ b/tests/mpi/mpi_blocked_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/mpi/mpi_sgemm/kernel.cpp b/tests/mpi/mpi_sgemm/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/mpi/mpi_sgemm/kernel.cpp +++ b/tests/mpi/mpi_sgemm/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/opencl/bfs/CLHelper.h b/tests/opencl/bfs/CLHelper.h index e4106845bd..eafce0d076 100755 --- a/tests/opencl/bfs/CLHelper.h +++ b/tests/opencl/bfs/CLHelper.h @@ -254,7 +254,6 @@ free(allPlatforms);*/ // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; if (0 != read_kernel_file("kernel.cl", &kernel_bin, &kernel_size)) std::abort(); @@ -374,7 +373,7 @@ free(allPlatforms);*/ void _clRelease() { char errorFlag = false; - for (int nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { + for (size_t nKernel = 0; nKernel < oclHandles.kernel.size(); nKernel++) { if (oclHandles.kernel[nKernel] != NULL) { cl_int resultCL = clReleaseKernel(oclHandles.kernel[nKernel]); if (resultCL != CL_SUCCESS) { @@ -511,7 +510,9 @@ cl_mem _clCreateAndCpyPinnedMem(int size, float *h_mem_source) { throw(string("excpetion in _clCreateAndCpyMem() -> clEnqueueMapBuffer")); #endif int element_number = size / sizeof(float); +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i = 0; i < element_number; i++) { h_mem_pinned[i] = h_mem_source[i]; } @@ -690,8 +691,8 @@ void _clInvokeKernel(int kernel_id, int work_items, // cannot be divided by work_group_size work_items = work_items + (work_group_size - (work_items % work_group_size)); - size_t local_work_size[] = {work_group_size, 1}; - size_t global_work_size[] = {work_items, 1}; + size_t local_work_size[] = {static_cast(work_group_size), 1}; + size_t global_work_size[] = {static_cast(work_items), 1}; oclHandles.cl_status = clEnqueueNDRangeKernel( oclHandles.queue, oclHandles.kernel[kernel_id], work_dim, 0, global_work_size, local_work_size, 0, 0, NULL); @@ -757,8 +758,10 @@ void _clInvokeKernel(int kernel_id, int work_items, void _clInvokeKernel2D(int kernel_id, int range_x, int range_y, int group_x, int group_y) { cl_uint work_dim = WORK_DIM; - size_t local_work_size[] = {group_x, group_y}; - size_t global_work_size[] = {range_x, range_y}; + size_t local_work_size[] = {static_cast(group_x), + static_cast(group_y)}; + size_t global_work_size[] = {static_cast(range_x), + static_cast(range_y)}; //cl_event e[1]; /*if(work_items%work_group_size != 0) //process situations that work_items cannot be divided by work_group_size diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index cd55f5b3fb..c676a40c53 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -29,10 +29,9 @@ struct Node { //--date: 26/01/2011 //--note: width is changed to the new_width //---------------------------------------------------------- -void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int edge_list_size, - int *h_graph_edges, char *h_graph_mask, - char *h_updating_graph_mask, char *h_graph_visited, - int *h_cost_ref) { +void run_bfs_cpu(int no_of_nodes, Node *h_graph_nodes, int *h_graph_edges, + char *h_graph_mask, char *h_updating_graph_mask, + char *h_graph_visited, int *h_cost_ref) { char stop; int k = 0; do { @@ -186,8 +185,9 @@ int main(int argc, char *argv[]) { int no_of_nodes; int edge_list_size; FILE *fp; - Node *h_graph_nodes; - char *h_graph_mask, *h_updating_graph_mask, *h_graph_visited; + Node *h_graph_nodes = NULL; + char *h_graph_mask = NULL, *h_updating_graph_mask = NULL, + *h_graph_visited = NULL; if (argc < 2) { printf("graph file missing!\n"); @@ -206,17 +206,22 @@ int main(int argc, char *argv[]) { printf("Reading File completed!\n"); + auto read_int = [fp](int *value) { + if (fscanf(fp, "%d", value) != 1) { + fclose(fp); + throw(std::string("error parsing graph file")); + } + }; + int source = 0; - fscanf(fp, "%d", &no_of_nodes); + read_int(&no_of_nodes); - int num_of_blocks = 1; int num_of_threads_per_block = no_of_nodes; // Make execution Parameters according to the number of nodes // Distribute threads across multiple Blocks if necessary if (no_of_nodes > MAX_THREADS_PER_BLOCK) { - num_of_blocks = (int)ceil(no_of_nodes / (double)MAX_THREADS_PER_BLOCK); num_of_threads_per_block = MAX_THREADS_PER_BLOCK; } work_group_size = num_of_threads_per_block; @@ -229,7 +234,8 @@ int main(int argc, char *argv[]) { int start, edgeno; // initalize the memory for (int i = 0; i < no_of_nodes; i++) { - fscanf(fp, "%d %d", &start, &edgeno); + read_int(&start); + read_int(&edgeno); h_graph_nodes[i].starting = start; h_graph_nodes[i].no_of_edges = edgeno; h_graph_mask[i] = false; @@ -237,17 +243,17 @@ int main(int argc, char *argv[]) { h_graph_visited[i] = false; } // read the source node from the file - fscanf(fp, "%d", &source); + read_int(&source); source = 0; // set the source node as true in the mask h_graph_mask[source] = true; h_graph_visited[source] = true; - fscanf(fp, "%d", &edge_list_size); + read_int(&edge_list_size); int id, cost; int *h_graph_edges = (int *)malloc(sizeof(int) * edge_list_size); for (int i = 0; i < edge_list_size; i++) { - fscanf(fp, "%d", &id); - fscanf(fp, "%d", &cost); + read_int(&id); + read_int(&cost); h_graph_edges[i] = id; } @@ -278,7 +284,7 @@ int main(int argc, char *argv[]) { source = 0; h_graph_mask[source] = true; h_graph_visited[source] = true; - run_bfs_cpu(no_of_nodes, h_graph_nodes, edge_list_size, h_graph_edges, + run_bfs_cpu(no_of_nodes, h_graph_nodes, h_graph_edges, h_graph_mask, h_updating_graph_mask, h_graph_visited, h_cost_ref); //--------------------------------------------------------- @@ -290,7 +296,7 @@ int main(int argc, char *argv[]) { free(h_updating_graph_mask); free(h_graph_visited); } catch (std::string msg) { - printf("--cambine: exception in main ->%s\n", msg); + printf("--cambine: exception in main ->%s\n", msg.c_str()); // release host memory free(h_graph_nodes); free(h_graph_mask); diff --git a/tests/opencl/bfs/util.h b/tests/opencl/bfs/util.h index 4d1d3e27b9..0941d18071 100755 --- a/tests/opencl/bfs/util.h +++ b/tests/opencl/bfs/util.h @@ -35,7 +35,9 @@ template void verify_array(const datatype *cpuResults, const datatype *gpuResults, const int size){ char passed = true; +#ifdef _OPENMP #pragma omp parallel for +#endif for (int i=0; i MAX_RELATIVE_ERROR){ passed = false; diff --git a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp index 41209e3637..19b89b6c73 100644 --- a/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp +++ b/tests/opencl/blackscholes/oclBlackScholes_launcher.cpp @@ -37,9 +37,8 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char **argv){ +extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqParamCommandQueue, const char ** /*argv*/){ cl_int ciErrNum; - size_t kernelLength; /*shrLog("...loading BlackScholes.cl\n"); char *cPathAndName = shrFindFilePath("BlackScholes.cl", argv[0]); @@ -50,8 +49,6 @@ extern "C" void initBlackScholes(cl_context cxGPUContext, cl_command_queue cqPar shrLog("...creating BlackScholes program\n"); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; - cl_device_id device_id = oclGetFirstDev(cxGPUContext); ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); shrCheckError(ciErrNum, CL_SUCCESS); diff --git a/tests/opencl/blackscholes/shrUtils.cpp b/tests/opencl/blackscholes/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/blackscholes/shrUtils.cpp +++ b/tests/opencl/blackscholes/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/cfd/main.cc b/tests/opencl/cfd/main.cc index 330d68ec66..1d8d11c741 100644 --- a/tests/opencl/cfd/main.cc +++ b/tests/opencl/cfd/main.cc @@ -158,9 +158,8 @@ static inline float compute_pressure(float density, float density_energy, float static inline float compute_speed_of_sound(float density, float pressure) { return sqrtf(GAMMA * pressure / density); } -static inline void compute_flux_contribution(float density, FLOAT3 momentum, float density_energy, - float pressure, FLOAT3 velocity, - FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, +static inline void compute_flux_contribution(FLOAT3 momentum, float density_energy, float pressure, + FLOAT3 velocity, FLOAT3* fc_momentum_x, FLOAT3* fc_momentum_y, FLOAT3* fc_momentum_z, FLOAT3* fc_density_energy) { fc_momentum_x->x = velocity.x * momentum.x + pressure; fc_momentum_x->y = velocity.x * momentum.y; @@ -216,7 +215,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, float pressure_i = compute_pressure(density_i, density_energy_i, speed_sqd_i); float speed_of_sound_i = compute_speed_of_sound(density_i, pressure_i); FLOAT3 fc_i_momentum_x, fc_i_momentum_y, fc_i_momentum_z, fc_i_density_energy; - compute_flux_contribution(density_i, momentum_i, density_energy_i, pressure_i, velocity_i, + compute_flux_contribution(momentum_i, density_energy_i, pressure_i, velocity_i, &fc_i_momentum_x, &fc_i_momentum_y, &fc_i_momentum_z, &fc_i_density_energy); float flux_i_density = 0.0f; @@ -244,7 +243,7 @@ static void cpu_compute_flux(int nelr, const int* esse, const float* normals, speed_sqd_nb = compute_speed_sqd(velocity_nb); pressure_nb = compute_pressure(density_nb, density_energy_nb, speed_sqd_nb); speed_of_sound_nb = compute_speed_of_sound(density_nb, pressure_nb); - compute_flux_contribution(density_nb, momentum_nb, density_energy_nb, pressure_nb, velocity_nb, + compute_flux_contribution(momentum_nb, density_energy_nb, pressure_nb, velocity_nb, &fc_nb_momentum_x, &fc_nb_momentum_y, &fc_nb_momentum_z, &fc_nb_density_energy); factor = -normal_len * smoothing_coefficient * 0.5f * @@ -352,8 +351,7 @@ int main(int argc, char** argv) { ff_momentum.x = h_ff_variable[VAR_MOMENTUM + 0]; ff_momentum.y = h_ff_variable[VAR_MOMENTUM + 1]; ff_momentum.z = h_ff_variable[VAR_MOMENTUM + 2]; - compute_flux_contribution(h_ff_variable[VAR_DENSITY], ff_momentum, - h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, + compute_flux_contribution(ff_momentum, h_ff_variable[VAR_DENSITY_ENERGY], ff_pressure, ff_velocity, &h_ff_fc_momentum_x, &h_ff_fc_momentum_y, &h_ff_fc_momentum_z, &h_ff_fc_density_energy); } diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index e35673e00e..84774d3e2d 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -51,6 +51,7 @@ VX_CFLAGS += -Xclang -target-feature -Xclang +xvortex VX_CFLAGS += -Xclang -target-feature -Xclang +zicond VX_LDFLAGS += -fuse-ld=lld VX_CFLAGS += -mllvm -disable-loop-idiom-all # disable memset/memcpy loop replacement +VX_CFLAGS += -Wall -Wextra -Wfatal-errors -Werror VX_LDFLAGS += -Wl,-z,norelro #VX_CFLAGS += -mllvm -vortex-branch-divergence=0 #VX_CFLAGS += -mllvm -debug -mllvm -print-after-all @@ -59,10 +60,11 @@ VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T$(VORTEX_HOME)/sw/kernel/scripts/link VX_BINTOOL += OBJCOPY=$(LLVM_PATH)/bin/llvm-objcopy $(VORTEX_HOME)/sw/kernel/scripts/vxbin.py -CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -CXXFLAGS += -Wno-deprecated-declarations -Wno-unused-parameter -Wno-narrowing +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror CXXFLAGS += -pthread CXXFLAGS += -I$(POCL_PATH)/include +# The tests are written against the OpenCL 1.2 host API. +CXXFLAGS += -DCL_TARGET_OPENCL_VERSION=120 CXXFLAGS += $(CONFIGS) POCL_CC_FLAGS += LLVM_PREFIX=$(LLVM_PATH) POCL_VORTEX_BINTOOL="$(VX_BINTOOL)" POCL_VORTEX_CFLAGS="$(VX_CFLAGS)" POCL_VORTEX_LDFLAGS="$(VX_LDFLAGS)" @@ -99,8 +101,9 @@ all: $(PROJECT) %.cpp.o: $(SRC_DIR)/%.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ +# Host C sources share the C++ flags except the language standard. %.c.o: $(SRC_DIR)/%.c - $(CC) $(CXXFLAGS) -c $< -o $@ + $(CC) $(filter-out -std=c++%,$(CXXFLAGS)) -c $< -o $@ $(VORTEX_KN_PATH)/libvortex.a: $(MAKE) -C $(VORTEX_KN_PATH) diff --git a/tests/opencl/conv3/main.cc b/tests/opencl/conv3/main.cc index 1220dabdb2..44bfe64886 100644 --- a/tests/opencl/conv3/main.cc +++ b/tests/opencl/conv3/main.cc @@ -180,7 +180,7 @@ int main (int argc, char **argv) { // Create kernel kernel = CL_CHECK2(clCreateKernel(program, KERNEL_NAME, &_err)); - size_t global_size[2] = {size, size}; + size_t global_size[2] = {static_cast(size), static_cast(size)}; size_t local_size[2] = {1, 1}; // Set kernel arguments diff --git a/tests/opencl/dotproduct/main.cc b/tests/opencl/dotproduct/main.cc index 789b41491b..fe4418228b 100644 --- a/tests/opencl/dotproduct/main.cc +++ b/tests/opencl/dotproduct/main.cc @@ -165,7 +165,6 @@ int main(int argc, char **argv) // Create the program shrLog("clCreateProgramWithSource...\n"); - cl_int binary_status; cl_program program; program = clCreateProgramWithSource( diff --git a/tests/opencl/dotproduct/shrUtils.cpp b/tests/opencl/dotproduct/shrUtils.cpp index cf0d2c3e7e..fb4bea436f 100644 --- a/tests/opencl/dotproduct/shrUtils.cpp +++ b/tests/opencl/dotproduct/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/opencl/guassian/clutils.cpp b/tests/opencl/guassian/clutils.cpp index 2881028489..53a25ff70d 100755 --- a/tests/opencl/guassian/clutils.cpp +++ b/tests/opencl/guassian/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -260,9 +252,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -401,7 +391,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -644,7 +633,11 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -726,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -833,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -879,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1421,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/guassian/main.cc b/tests/opencl/guassian/main.cc index fbb99576e4..a06ebbbdcd 100755 --- a/tests/opencl/guassian/main.cc +++ b/tests/opencl/guassian/main.cc @@ -31,7 +31,10 @@ int main(int argc, char *argv[]) { if (size == 0) { fp = fopen(filename, "r"); - fscanf(fp, "%d", &size); + if (fscanf(fp, "%d", &size) != 1) { + printf("Error reading matrix size from %s\n", filename); + exit(1); + } printf("using %dx%d input matrix\n", size, size); @@ -200,8 +203,9 @@ void ForwardSub(cl_context context, float *a, float *b, float *m, int size, writeMB = (float)(sizeof(float) * size * (size + size + 1) / 1e6); // 3. Determine block sizes - size_t globalWorksizeFan1[1] = {size}; - size_t globalWorksizeFan2[2] = {size, size}; + size_t globalWorksizeFan1[1] = {static_cast(size)}; + size_t globalWorksizeFan2[2] = {static_cast(size), + static_cast(size)}; size_t localWorksizeFan1[1] = {1}; size_t localWorksizeFan2[2] = {1, 1}; @@ -425,7 +429,10 @@ void InitMat(FILE *fp, int size, float *ary, int nrow, int ncol) { for (i = 0; i < nrow; i++) { for (j = 0; j < ncol; j++) { - fscanf(fp, "%f", ary + size * i + j); + if (fscanf(fp, "%f", ary + size * i + j) != 1) { + printf("Error reading input matrix\n"); + exit(1); + } } } } @@ -438,7 +445,10 @@ void InitAry(FILE *fp, float *ary, int ary_size) { int i; for (i = 0; i < ary_size; i++) { - fscanf(fp, "%f", &ary[i]); + if (fscanf(fp, "%f", &ary[i]) != 1) { + printf("Error reading input array\n"); + exit(1); + } } } /*------------------------------------------------------ diff --git a/tests/opencl/image_get/main.cc b/tests/opencl/image_get/main.cc index 8ceae1749f..3ec48fb636 100644 --- a/tests/opencl/image_get/main.cc +++ b/tests/opencl/image_get/main.cc @@ -21,9 +21,12 @@ static int read_kernel_file(const char* fn, uint8_t** data, size_t* size) { cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue = NULL; cl_program program = NULL; cl_kernel kernel = NULL; cl_mem image = NULL, out_buf = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (image) clReleaseMemObject(image); - if (out_buf) clReleaseMemObject(out_buf); if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (image) { clReleaseMemObject(image); } + if (out_buf) { clReleaseMemObject(out_buf); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { diff --git a/tests/opencl/image_int/main.cc b/tests/opencl/image_int/main.cc index 2665fa6ee5..af04df8148 100644 --- a/tests/opencl/image_int/main.cc +++ b/tests/opencl/image_int/main.cc @@ -23,10 +23,13 @@ cl_device_id device_id = NULL; cl_context context = NULL; cl_command_queue queue cl_program program = NULL; cl_kernel kernel = NULL; cl_sampler sampler = NULL; cl_mem src_image = NULL, dst_image = NULL; static void cleanup() { - if (queue) clReleaseCommandQueue(queue); if (kernel) clReleaseKernel(kernel); - if (program) clReleaseProgram(program); if (sampler) clReleaseSampler(sampler); - if (src_image) clReleaseMemObject(src_image); if (dst_image) clReleaseMemObject(dst_image); - if (context) clReleaseContext(context); + if (queue) { clReleaseCommandQueue(queue); } + if (kernel) { clReleaseKernel(kernel); } + if (program) { clReleaseProgram(program); } + if (sampler) { clReleaseSampler(sampler); } + if (src_image) { clReleaseMemObject(src_image); } + if (dst_image) { clReleaseMemObject(dst_image); } + if (context) { clReleaseContext(context); } } int main(int argc, char** argv) { @@ -66,7 +69,11 @@ int main(int argc, char** argv) { int errors=0; for (size_t i=0;i(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE; // work group size is defined by // RD_WG_SIZE_0 or RD_WG_SIZE_0_0 @@ -335,10 +332,10 @@ int kmeansOCL(float **feature, /* in: [npoints][nfeatures] */ float **clusters, int *new_centers_len, float **new_centers) { int delta = 0; - int i, j, k; + int i, j; cl_int err = 0; - size_t global_work[3] = {n_points, 1, 1}; + size_t global_work[3] = {static_cast(n_points), 1, 1}; /// Ke Wang adjustable local group size 2013/08/07 10:37:33 size_t local_work_size = BLOCK_SIZE2; // work group size is defined by diff --git a/tests/opencl/kmeans/read_input.c b/tests/opencl/kmeans/read_input.c index f70e7fb300..1dfc2df346 100755 --- a/tests/opencl/kmeans/read_input.c +++ b/tests/opencl/kmeans/read_input.c @@ -126,7 +126,6 @@ int setup(int argc, char **argv) { int best_nclusters = 0; int nfeatures = 16; /* small self-contained default */ int npoints = 512; /* small self-contained default */ - float len; float **features; float **cluster_centres = NULL; @@ -192,8 +191,11 @@ int setup(int argc, char **argv) { fprintf(stderr, "Error: no such file (%s)\n", filename); exit(1); } - fread(&npoints, 1, sizeof(int), infile); - fread(&nfeatures, 1, sizeof(int), infile); + if (fread(&npoints, 1, sizeof(int), infile) != sizeof(int) || + fread(&nfeatures, 1, sizeof(int), infile) != sizeof(int)) { + fprintf(stderr, "Error: failed to read header of (%s)\n", filename); + exit(1); + } // allocate space for features[][] and read attributes of all objects buf = (float *)malloc(npoints * nfeatures * sizeof(float)); @@ -202,7 +204,11 @@ int setup(int argc, char **argv) { for (i = 1; i < npoints; i++) { features[i] = features[i - 1] + nfeatures; } - fread(buf, 1, npoints * nfeatures * sizeof(float), infile); + if (fread(buf, 1, npoints * nfeatures * sizeof(float), infile) != + npoints * nfeatures * sizeof(float)) { + fprintf(stderr, "Error: failed to read features of (%s)\n", filename); + exit(1); + } fclose(infile); } else { FILE *infile; @@ -320,8 +326,6 @@ int setup(int argc, char **argv) { } } - len = (float)((max_nclusters - min_nclusters + 1) * nloops); - printf("Number of Iteration: %d\n", nloops); // printf("Time for I/O: %.5fsec\n", io_timing); // printf("Time for Entire Clustering: %.5fsec\n", cluster_timing); diff --git a/tests/opencl/kmeans/rmse.c b/tests/opencl/kmeans/rmse.c index 03d614a624..2f6cee3c61 100755 --- a/tests/opencl/kmeans/rmse.c +++ b/tests/opencl/kmeans/rmse.c @@ -42,7 +42,7 @@ int find_nearest_point(float *pt, /* [nfeatures] */ float **pts, /* [npts][nfeatures] */ int npts) { - int index, i; + int index = 0, i; float max_dist=FLT_MAX; /* find the cluster center id with min distance to pt */ @@ -70,11 +70,13 @@ float rms_err (float **feature, /* [npoints][nfeatures] */ float ret; /* return value */ /* calculate and sum the sqaure of euclidean distance*/ +#ifdef _OPENMP #pragma omp parallel for \ shared(feature,cluster_centres) \ firstprivate(npoints,nfeatures,nclusters) \ private(i, nearest_cluster_index) \ - schedule (static) + schedule (static) +#endif for (i=0; iasync_markers != NULL) && - (timers->async_markers->timerID != INVALID_TIMERID); -} - -static struct pb_async_time_marker_list * -get_last_async(struct pb_TimerSet* timers) -{ - /* Find the last event recorded thus far */ - struct pb_async_time_marker_list * last_event = timers->async_markers; - if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { - while(last_event->next != NULL && - last_event->next->timerID != INVALID_TIMERID) - last_event = last_event->next; - return last_event; - } else - return NULL; -} - -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - -static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = label; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - - -/* Assumes that all recorded events have completed */ -static pb_Timestamp record_async_times(struct pb_TimerSet* tset) -{ - struct pb_async_time_marker_list * next_interval = NULL; - struct pb_async_time_marker_list * last_marker = get_last_async(tset); - pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; - - for(next_interval = tset->async_markers; next_interval != last_marker; - next_interval = next_interval->next) { - cl_ulong command_start=0, command_end=0; - cl_int ciErrNum = CL_SUCCESS; - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); - } - - ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); - } - - pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); - tset->timers[next_interval->timerID].elapsed += interval; - if (next_interval->label != NULL) { - struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; - while (subtimer != NULL) { - if ( strcmp(subtimer->label, next_interval->label) == 0) { - subtimer->timer.elapsed += interval; - break; - } - subtimer = subtimer->next; - } - } - total_async_time += interval; - next_interval->timerID = INVALID_TIMERID; - } - - if(next_interval != NULL) - next_interval->timerID = INVALID_TIMERID; - - return total_async_time; -} - -static void -accumulate_time(pb_Timestamp *accum, - pb_Timestamp start, - pb_Timestamp end) -{ -//#if _POSIX_VERSION >= 200112L - *accum += end - start; -//#else -//# error "Timestamps not implemented for this system" -//#endif -} - -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +40,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +64,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +108,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +137,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +188,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +209,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -406,6 +232,9 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { void pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +269,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -560,6 +391,9 @@ pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) void pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) { + (void)timers; + (void)label; + (void)category; #if 0 #ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; @@ -708,6 +542,7 @@ pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID cat void pb_PrintTimerSet(struct pb_TimerSet *timers) { + (void)timers; #if 0 #ifndef DISABLE_PARBOIL_TIMER pb_Timestamp wall_end = 0; //get_time(); @@ -1198,6 +1033,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1254,11 +1090,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1370,7 +1208,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/myocyte/myocyte_model.h b/tests/opencl/myocyte/myocyte_model.h index e4212ecc55..2aff0285f7 100644 --- a/tests/opencl/myocyte/myocyte_model.h +++ b/tests/opencl/myocyte/myocyte_model.h @@ -922,7 +922,7 @@ kernel_ecc( fp timeinst, case 2: V_hold = -55; V_test = 0; - if(timeinst>0.5 & timeinst<200.5){ + if((timeinst>0.5) & (timeinst<200.5)){ V_clamp = V_test; } else{ @@ -951,7 +951,7 @@ kernel_ecc( fp timeinst, //========================================================================================================================================================================================================200 void -kernel_cam( fp timeinst, +kernel_cam( fp /*timeinst*/, GLOBAL fp *d_initvalu, GLOBAL fp *d_finavalu, int valu_offset, diff --git a/tests/opencl/nearn/clutils.cpp b/tests/opencl/nearn/clutils.cpp index b411b19b15..8cd8065195 100755 --- a/tests/opencl/nearn/clutils.cpp +++ b/tests/opencl/nearn/clutils.cpp @@ -71,9 +71,6 @@ static cl_uint numPlatforms; static cl_device_id* devices = NULL; static cl_uint* numDevices = NULL; -//! The chosen OpenCL platform -static cl_platform_id platform = NULL; - //! The chosen OpenCL device static cl_device_id device = NULL; @@ -82,11 +79,6 @@ static cl_context context = NULL; //! OpenCL command queue static cl_command_queue commandQueue = NULL; -static cl_command_queue commandQueueProf = NULL; -static cl_command_queue commandQueueNoProf = NULL; - -//! Global status of events -static bool eventsEnabled = false; //------------------------------------------------------- // Initialization and Cleanup @@ -259,9 +251,7 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return 0; } -cl_context cl_init_context(int platform, int dev,int quiet) { - int printInfo=1; - if (platform >= 0 && dev >= 0) printInfo = 0; +cl_context cl_init_context(int /*platform*/, int /*dev*/, int /*quiet*/) { cl_int status; // Used to iterate through the platforms and devices, respectively @@ -400,7 +390,6 @@ cl_context cl_init_context(int platform, int dev,int quiet) { numDevices[0] = 1; devices = (cl_device_id*)malloc(sizeof(cl_device_id)*numDevices[0]); - int platform_touse = 0; int device_touse = 0; status = clGetPlatformIDs(numPlatforms, platforms, NULL); @@ -643,7 +632,12 @@ cl_mem cl_allocImage(size_t height, size_t width, char type, cl_mem_flags flags) allocationSize += height*width*elemSize; // Create the image - mem = clCreateImage2D(context, flags, &format, width, height, 0, NULL, &status); + cl_image_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + mem = clCreateImage(context, flags, &format, &desc, NULL, &status); //cl_errChk(status, "creating image", true); if(status != CL_SUCCESS) { @@ -725,7 +719,7 @@ void cl_copyBufferToHost(void* dst, cl_mem src, size_t mem_size, cl_bool blockin void cl_copyBufferToImage(cl_mem buffer, cl_mem image, int height, int width) { size_t origin[3] = {0, 0, 0}; - size_t region[3] = {width, height, 1}; + size_t region[3] = {static_cast(width), static_cast(height), 1}; cl_int status; status = clEnqueueCopyBufferToImage(commandQueue, buffer, image, 0, @@ -832,12 +826,9 @@ Compile Opencl source file into a cl_program. The cl_program will be made into a \param compileoptions Compilation options \param verbosebuild Switch to enable verbose Output */ -cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbosebuild ) +cl_program cl_compileProgram(char* /*kernelPath*/, char* compileoptions, bool verbosebuild ) { cl_int status; - FILE *fp = NULL; - char *source = NULL; - long int size; /*printf("\t%s\n", kernelPath); @@ -878,7 +869,6 @@ cl_program cl_compileProgram(char* kernelPath, char* compileoptions, bool verbos // read kernel binary from file uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgramReturn; // Create the program object @@ -1420,8 +1410,7 @@ char* catStringWithInt(const char* string, int integer) { char tmp[6]; - strcpy(eventStr, string); - strncat(eventStr, itoa_portable(integer, tmp, 10), 5); + snprintf(eventStr, strLen, "%s%s", string, itoa_portable(integer, tmp, 10)); return eventStr; } diff --git a/tests/opencl/nearn/main.cc b/tests/opencl/nearn/main.cc index 973dcd9c8c..ea8238e167 100755 --- a/tests/opencl/nearn/main.cc +++ b/tests/opencl/nearn/main.cc @@ -209,8 +209,6 @@ int loadData(char *filename, std::vector &records, /**Main processing **/ - int q = 0; - flist = fopen(filename, "r"); while (!feof(flist)) { /** @@ -232,7 +230,13 @@ int loadData(char *filename, std::vector &records, while (!feof(fp)) { Record record; LatLong latLong; - fgets(record.recString, 49, fp); + if (fgets(record.recString, 49, fp) == NULL) { + if (ferror(fp)) { + printf("error reading a db\n"); + exit(1); + } + break; + } fgetc(fp); // newline if (feof(fp)) break; diff --git a/tests/opencl/oclprintf/main.cc b/tests/opencl/oclprintf/main.cc index c23e6dec08..73dc536716 100644 --- a/tests/opencl/oclprintf/main.cc +++ b/tests/opencl/oclprintf/main.cc @@ -147,7 +147,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 8ecfdc5239..0e72599ece 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -167,7 +167,7 @@ int main (int argc, char **argv) { CL_CHECK(clEnqueueWriteBuffer(commandQueue, a_memobj, CL_TRUE, 0, nbytes, h_a, 0, NULL, NULL)); printf("Execute the kernel\n"); - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; auto time_start = std::chrono::high_resolution_clock::now(); CL_CHECK(clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, NULL)); diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 2397c720e4..2cb4b71ba8 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -53,8 +53,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -198,7 +198,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 2, sizeof(factor), &factor)); size_t global_offset[1] = {0}; - size_t global_work_size[1] = {size}; + size_t global_work_size[1] = {static_cast(size)}; size_t local_work_size[1] = {1}; printf("initialize buffers\n"); diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index 97cfb689e7..c56d26ffdc 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -56,8 +56,8 @@ _ret; \ }) -void pfn_notify(const char *errinfo, const void *private_info, size_t cb, - void *user_data) { +void pfn_notify(const char *errinfo, const void * /*private_info*/, size_t /*cb*/, + void * /*user_data*/) { fprintf(stderr, "OpenCL Error (via pfn_notify): %s\n", errinfo); } @@ -220,7 +220,7 @@ int main(int argc, char **argv) { CL_CHECK(clSetKernelArg(kernel, 11, sizeof(m8), (&m8))); size_t global_offset[2] = {1, 1}; - size_t global_work_size[2] = {size - 2, size - 2}; + size_t global_work_size[2] = {static_cast(size - 2), static_cast(size - 2)}; size_t local_work_size[2] = {1, 1}; // {size-2,1} printf("enqueue write buffer\n"); diff --git a/tests/opencl/spmv/args.c b/tests/opencl/spmv/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/spmv/args.c +++ b/tests/opencl/spmv/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/spmv/convert_dataset.c b/tests/opencl/spmv/convert_dataset.c index aba9c3b3de..8aab7b32d4 100644 --- a/tests/opencl/spmv/convert_dataset.c +++ b/tests/opencl/spmv/convert_dataset.c @@ -83,7 +83,6 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, FILE *f; int nz; int i; - float *val; mat_entry *entries; row_stats *stats; int rows, cols; @@ -126,10 +125,16 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, for (i = 0; i < nz; i++, cur_i++) { if (!binary) { - fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, - &entries[cur_i].val); + if (fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, + &entries[cur_i].val) != 3) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } } else { - fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col); + if (fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col) != 2) { + printf("Could not read matrix entry %d.\n", i); + exit(1); + } entries[cur_i].val = 1.0; } entries[cur_i].row--; @@ -216,7 +221,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, if (debug_level >= 1) { printf("Padding data....%d rows, %d groups\n", rows, *nz_count_len); } - int pad_to, total_padding = 0, pack_to; + int pad_to = 0, total_padding = 0, pack_to; pad_rows *= pack_size; // change padding to account for packed items for (i = 0; i < rows; i++) { // record JDS to real row number @@ -281,7 +286,7 @@ int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, */ irow = 0; // keep track of which row we are in inside the fubini-ed array int idata = 0; // position within final data array - int entry_index, j; + int entry_index; int ipack; // used in internal loop for writing packed values mat_entry entry; while (1) { diff --git a/tests/opencl/spmv/file.c b/tests/opencl/spmv/file.c index a98451e56b..23709bad19 100644 --- a/tests/opencl/spmv/file.c +++ b/tests/opencl/spmv/file.c @@ -28,27 +28,50 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p exit(-1); } - fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad); + if (fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad) != 5) + { + fprintf(stderr, "Error reading input file header\n"); + exit(-1); + } int _len=len[0]; int _depth=depth[0]; int _dim=dim[0]; - int _pad=pad[0]; int _nzcnt_len=nzcnt_len[0]; *h_data = (float *) malloc(_len * sizeof (float)); - fread (*h_data, sizeof (float), _len, fid); + if (fread (*h_data, sizeof (float), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file data\n"); + exit(-1); + } *h_indices = (int *) malloc(_len * sizeof (int)); - fread (*h_indices, sizeof (int), _len, fid); + if (fread (*h_indices, sizeof (int), _len, fid) != (size_t)_len) + { + fprintf(stderr, "Error reading input file indices\n"); + exit(-1); + } *h_ptr = (int *) malloc(_depth * sizeof (int)); - fread (*h_ptr, sizeof (int), _depth, fid); + if (fread (*h_ptr, sizeof (int), _depth, fid) != (size_t)_depth) + { + fprintf(stderr, "Error reading input file row pointers\n"); + exit(-1); + } *h_perm = (int *) malloc(_dim * sizeof (int)); - fread (*h_perm, sizeof (int), _dim, fid); + if (fread (*h_perm, sizeof (int), _dim, fid) != (size_t)_dim) + { + fprintf(stderr, "Error reading input file permutation\n"); + exit(-1); + } *h_nzcnt = (int *) malloc(_nzcnt_len * sizeof (int)); - fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid); + if (fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid) != (size_t)_nzcnt_len) + { + fprintf(stderr, "Error reading input file nonzero counts\n"); + exit(-1); + } fclose (fid); } @@ -56,7 +79,11 @@ void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *p void input_vec(char *fName,float *h_vec,int dim) { FILE* fid = fopen(fName, "rb"); - fread (h_vec, sizeof (float), dim, fid); + if (fread (h_vec, sizeof (float), dim, fid) != (size_t)dim) + { + fprintf(stderr, "Error reading input vector file\n"); + exit(-1); + } fclose(fid); } diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index 0be7565cd0..fada19c156 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -50,7 +50,7 @@ static float* read_output_file(const char* filename, int* out_size) { return NULL; } // Read the float data - if (fread(floats, sizeof(float), size, file) != size) { + if (fread(floats, sizeof(float), size, file) != (size_t)size) { fclose(file); free(floats); perror("Error reading floats from file"); @@ -109,15 +109,6 @@ static int read_kernel_file(const char* filename, uint8_t** data, size_t* size) return CL_SUCCESS; } -static int generate_vector(float *x_vector, int dim) { - srand(54321); - int i; - for (i = 0; i < dim; i++) { - x_vector[i] = (rand() / (float)RAND_MAX); - } - return 0; -} - int main(int argc, char **argv) { struct pb_TimerSet timers; struct pb_Parameters *parameters; @@ -146,7 +137,6 @@ int main(int argc, char **argv) { } cl_device_id clDevice = (cl_device_id)pb_context->clDeviceId; - cl_platform_id clPlatform = (cl_platform_id)pb_context->clPlatformId; cl_context clContext = (cl_context)pb_context->clContext; cl_command_queue clCommandQueue = clCreateCommandQueue( clContext, clDevice, CL_QUEUE_PROFILING_ENABLE, &clStatus); @@ -159,7 +149,6 @@ int main(int argc, char **argv) { // clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -169,7 +158,7 @@ int main(int argc, char **argv) { CHECK_ERROR("clCreateProgramWithSource") char clOptions[50]; - sprintf(clOptions, ""); + clOptions[0] = '\0'; clStatus = clBuildProgram(clProgram, 1, &clDevice, clOptions, NULL, NULL); CHECK_ERROR("clBuildProgram") diff --git a/tests/opencl/spmv/mmio.c b/tests/opencl/spmv/mmio.c index a356471851..c5eb01eb6f 100644 --- a/tests/opencl/spmv/mmio.c +++ b/tests/opencl/spmv/mmio.c @@ -72,7 +72,12 @@ int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_, for (i=0; i #endif -#include "perfmon.h" cl_context *clContextPtr; cl_command_queue *clCommandQueuePtr; @@ -56,46 +55,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +87,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +101,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +151,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +168,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +192,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +236,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +265,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +316,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +337,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +359,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +397,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +691,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1154,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1196,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1314,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/srad/main.cc b/tests/opencl/srad/main.cc index 3ef2cfa1a1..c15d1deb8f 100644 --- a/tests/opencl/srad/main.cc +++ b/tests/opencl/srad/main.cc @@ -129,7 +129,7 @@ static void parse_args(int argc, char** argv) { // Serial CPU reference: identical single-precision SRAD math over the same // (extracted) image. Mirrors extract -> [stats, srad, srad2]*niter -> compress. -static void srad_cpu(std::vector& I, int Nr, int Nc, long Ne, int niter, +static void srad_cpu(std::vector& I, int Nr, long Ne, int niter, float lambda, long NeROI, const std::vector& iN, const std::vector& iS, const std::vector& jE, const std::vector& jW) { @@ -342,7 +342,7 @@ int main(int argc, char** argv) { // CPU golden reference over the same seeded input. std::vector h_ref(h_image); - srad_cpu(h_ref, Nr, Nc, Ne, niter, lambda, NeROI, iN, iS, jE, jW); + srad_cpu(h_ref, Nr, Ne, niter, lambda, NeROI, iN, iS, jE, jW); // Compare with a mixed absolute/relative float tolerance (exp/log + iterated // accumulation across host and device libm differ by a few ULP). diff --git a/tests/opencl/stencil/args.c b/tests/opencl/stencil/args.c index 9d751e2940..42637fe1c5 100644 --- a/tests/opencl/stencil/args.c +++ b/tests/opencl/stencil/args.c @@ -225,7 +225,7 @@ read_platform_param(char *str) /* The platform name is either the entire string, or all characters before * the separator */ - int name_length = separator_index == -1 ? strlen(str) : separator_index; + int name_length = separator_index == -1 ? (int)strlen(str) : separator_index; char *name_str = (char *)malloc(name_length + 1); memcpy(name_str, str, name_length); name_str[name_length] = 0; @@ -410,6 +410,8 @@ parse_input_files(struct argparse *ap, struct pb_Parameters *params) static result parse_end_options(struct argparse *ap, struct pb_Parameters *params) { + (void)ap; + (void)params; return ARGPARSE_DONE; } @@ -500,7 +502,6 @@ is_last_option(struct option *op) static int pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) { - char *err_message; struct argparse ap; /* Each argument */ diff --git a/tests/opencl/stencil/file.c b/tests/opencl/stencil/file.c index a9b39b1935..4b44376df6 100644 --- a/tests/opencl/stencil/file.c +++ b/tests/opencl/stencil/file.c @@ -26,9 +26,13 @@ void inputData(char* fName, int* nx, int* ny, int* nz) exit(-1); } - fread(nx, sizeof(int ),1,fid); - fread(ny, sizeof(int ),1,fid); - fread(nz, sizeof(int ),1,fid); + if (fread(nx, sizeof(int ),1,fid) != 1 || + fread(ny, sizeof(int ),1,fid) != 1 || + fread(nz, sizeof(int ),1,fid) != 1) + { + fprintf(stderr, "Error reading input file\n"); + exit(-1); + } fclose (fid); } diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index 0018d00d70..30dee81bd0 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -35,7 +35,10 @@ static int read_data(float *A0, int nx,int ny,int nz,FILE *fp) { for(k=0;kclPlatformId; clContext = (cl_context) pb_context->clContext; clDevice = (cl_device_id) pb_context->clDeviceId; @@ -208,7 +209,6 @@ int main(int argc, char** argv) { //cl_program clProgram = clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; cl_program clProgram; clStatus = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); @@ -241,7 +241,10 @@ int main(int argc, char** argv) { h_Anext=(float*)malloc(sizeof(float)*size); pb_SwitchToTimer(&timers, pb_TimerID_IO); FILE *fp = fopen(parameters->inpFiles[0], "rb"); - read_data(h_A0, nx,ny,nz, fp); + if (read_data(h_A0, nx,ny,nz, fp) != 0) { + fclose(fp); + return -1; + } fclose(fp); memcpy (h_Anext,h_A0,sizeof(float)*size); @@ -264,9 +267,8 @@ int main(int argc, char** argv) { //only use 1D thread block int tx = 128; size_t block[3] = {1,1,1}; // {tx,1,1} - size_t grid[3] = {(nx-2+tx-1)/tx*tx,ny-2,nz-2}; + size_t grid[3] = {static_cast((nx-2+tx-1)/tx*tx),static_cast(ny-2),static_cast(nz-2)}; //size_t grid[3] = {nx-2,ny-2,nz-2}; - size_t offset[3] = {1,1,1}; printf("grid size in x/y/z = %ld %ld %ld\n",grid[0],grid[1],grid[2]); printf("block size in x/y/z = %ld %ld %ld\n",block[0],block[1],block[2]); diff --git a/tests/opencl/stencil/ocl.c b/tests/opencl/stencil/ocl.c index 9ce9a2f5d2..08b3aa06b5 100644 --- a/tests/opencl/stencil/ocl.c +++ b/tests/opencl/stencil/ocl.c @@ -27,7 +27,7 @@ char* readFile(const char* fileName) } size_t res = fread(buffer,1,size,fp); - if(res != size) + if(res != (size_t)size) { printf("Error 3!\n"); fclose(fp); diff --git a/tests/opencl/stencil/parboil_opencl.c b/tests/opencl/stencil/parboil_opencl.c index 7348dc2c0d..3caf4967de 100644 --- a/tests/opencl/stencil/parboil_opencl.c +++ b/tests/opencl/stencil/parboil_opencl.c @@ -56,46 +56,6 @@ get_last_async(struct pb_TimerSet* timers) return NULL; } -static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) -{ - cl_int ciErrNum = CL_SUCCESS; - struct pb_async_time_marker_list ** new_event = &(tset->async_markers); - - while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { - new_event = &((*new_event)->next); - } - - if(*new_event == NULL) { - *new_event = (struct pb_async_time_marker_list *) - malloc(sizeof(struct pb_async_time_marker_list)); - (*new_event)->marker = calloc(1, sizeof(cl_event)); - /* - // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event -#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) -fprintf(stderr, "Creating Marker [%d]\n", timer); - *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Creating User Event Object!\n"); - } - ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Setting User Event Status!\n"); - } -#endif -*/ - (*new_event)->next = NULL; - } - - /* valid event handle now aquired: insert the event record */ - (*new_event)->label = NULL; - (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); - if (ciErrNum != CL_SUCCESS) { - fprintf(stderr, "Error Enqueueing Marker!\n"); - } - -} - static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) { cl_int ciErrNum = CL_SUCCESS; @@ -128,7 +88,7 @@ fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); /* valid event handle now aquired: insert the event record */ (*new_event)->label = label; (*new_event)->timerID = timer; - ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + ciErrNum = clEnqueueMarkerWithWaitList(*clCommandQueuePtr, 0, NULL, (cl_event *)(*new_event)->marker); if (ciErrNum != CL_SUCCESS) { fprintf(stderr, "Error Enqueueing Marker!\n"); } @@ -142,7 +102,6 @@ static pb_Timestamp record_async_times(struct pb_TimerSet* tset) struct pb_async_time_marker_list * next_interval = NULL; struct pb_async_time_marker_list * last_marker = get_last_async(tset); pb_Timestamp total_async_time = 0; - enum pb_TimerID timer; for(next_interval = tset->async_markers; next_interval != last_marker; next_interval = next_interval->next) { @@ -193,18 +152,6 @@ accumulate_time(pb_Timestamp *accum, //#endif } -//#if _POSIX_VERSION >= 200112L -static pb_Timestamp get_time() -{ - //struct timeval tv; - //gettimeofday(&tv, NULL); - //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); - return 0; -} -//#else -//# error "no supported time libraries are available on this platform" -//#endif - void pb_ResetTimer(struct pb_Timer *timer) { @@ -222,6 +169,7 @@ pb_ResetTimer(struct pb_Timer *timer) void pb_StartTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER if (timer->state != pb_Timer_STOPPED) { fputs("Ignoring attempt to start a running timer\n", stderr); @@ -245,6 +193,8 @@ pb_StartTimer(struct pb_Timer *timer) void pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER unsigned int numNotStopped = 0x3; // 11 @@ -287,6 +237,7 @@ pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) void pb_StopTimer(struct pb_Timer *timer) { + (void)timer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -315,6 +266,8 @@ pb_StopTimer(struct pb_Timer *timer) } void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + (void)timer; + (void)subtimer; /*#ifndef DISABLE_PARBOIL_TIMER pb_Timestamp fini; @@ -364,6 +317,7 @@ void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) double pb_GetElapsedTime(struct pb_Timer *timer) { + (void)timer; /*double ret; #ifndef DISABLE_PARBOIL_TIMER @@ -384,6 +338,7 @@ pb_GetElapsedTime(struct pb_Timer *timer) void pb_InitializeTimerSet(struct pb_TimerSet *timers) { + (void)timers; /*#ifndef DISABLE_PARBOIL_TIMER int n; @@ -405,7 +360,10 @@ void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { } void -pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + (void)timers; + (void)label; + (void)pb_Category; /*#ifndef DISABLE_PARBOIL_TIMER struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc @@ -440,6 +398,8 @@ pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Categ void pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) { + (void)timers; + (void)timer; #if 0 #ifndef DISABLE_PARBOIL_TIMER @@ -732,8 +692,9 @@ pb_PrintTimerSet(struct pb_TimerSet *timers) maxSubLength = 0; while (sub != NULL) { // Find longest SubTimer label - if (strlen(sub->label) > maxSubLength) { - maxSubLength = strlen(sub->label); + int labelLength = (int)strlen(sub->label); + if (labelLength > maxSubLength) { + maxSubLength = labelLength; } sub = sub->next; } @@ -1194,6 +1155,7 @@ pb_GetPlatform(struct pb_PlatformParam *platform) { pb_Context* pb_InitOpenCLContext(struct pb_Parameters* parameters) { + (void)parameters; #if 0 pb_Platform* ps = pb_GetPlatform(parameters->platform); if (!ps) return NULL; @@ -1235,11 +1197,13 @@ pb_InitOpenCLContext(struct pb_Parameters* parameters) { void pb_ReleaseOpenCLContext(pb_Context* c) { + (void)c; pb_ReleasePlatforms(); } void pb_PrintPlatformInfo(pb_Context* c) { + (void)c; /*pb_Platform** ps = pb_GetPlatforms(); if (!ps) { fprintf (stderr, "No platform found"); @@ -1351,7 +1315,7 @@ pb_sig_int(char* c, int* p, int sz) { void pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { - int i; + unsigned int i; unsigned long long int s = 0; for (i = 0; i < sz; i++) s += p[i]; printf ("[Signature] %s = %lld\n", c, s); diff --git a/tests/opencl/transpose/main.cc b/tests/opencl/transpose/main.cc index 93c3a4fdbd..8a2148b4c3 100644 --- a/tests/opencl/transpose/main.cc +++ b/tests/opencl/transpose/main.cc @@ -62,7 +62,7 @@ static double transposeGPU(const char* kernelName, bool useLocalMem, cl_uint ci { cl_mem d_odata[MAX_GPU_COUNT]; cl_mem d_idata[MAX_GPU_COUNT]; - cl_kernel ckKernel[MAX_GPU_COUNT]; + cl_kernel ckKernel[MAX_GPU_COUNT] = {}; size_t szGlobalWorkSize[2]; size_t szLocalWorkSize[2]; @@ -317,7 +317,6 @@ int runTest( const int argc, const char** argv) // (const char **)&source, &program_length, &ciErrNum); uint8_t *kernel_bin = NULL; size_t kernel_size; - cl_int binary_status = 0; ciErrNum = read_kernel_file("kernel.cl", &kernel_bin, &kernel_size); if (ciErrNum != CL_SUCCESS) { @@ -349,7 +348,7 @@ int runTest( const int argc, const char** argv) //double sharedCopyTime = transposeGPU("shared_copy", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #endif - double naiveTime = transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); + transposeGPU("transpose_naive", false, ciDeviceCount, h_idata, h_odata, size_x, size_y); //double optimizedTime = transposeGPU("transpose", true, ciDeviceCount, h_idata, h_odata, size_x, size_y); #ifdef GPU_PROFILING diff --git a/tests/opencl/transpose/shrUtils.cpp b/tests/opencl/transpose/shrUtils.cpp index 23e4488d27..89c0b7169b 100644 --- a/tests/opencl/transpose/shrUtils.cpp +++ b/tests/opencl/transpose/shrUtils.cpp @@ -203,7 +203,6 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { static FILE* pFileStream0 = NULL; static FILE* pFileStream1 = NULL; - size_t szNumWritten = 0; char cFileMode [3]; // if the sample log file is closed and the call incudes a "write-to-file", open file for writing @@ -318,12 +317,12 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print string to console if flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = printf ("\n !!! Error # %i at ", iErrNum); // console + printf ("\n !!! Error # %i at ", iErrNum); // console } // print string to file if flagged if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file + fprintf (pFileStream0, "\n !!! Error # %i at ", iErrNum); // sample log file } } @@ -347,14 +346,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // character is not '%', so print it verbatim to console and/or files as flagged if (iLogMode & LOGCONSOLE) { - szNumWritten = putc(*pStr, stdout); // console + putc(*pStr, stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } @@ -399,16 +398,26 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list { case '%': // special handling for string of %%%% { + // collapse each "%%" to a literal '%', as printf would + std::string sLiteral; + for (size_t i = 0; i < sFormatSpec.size(); ++i) + { + sLiteral += sFormatSpec[i]; + if ((sFormatSpec[i] == '%') && (i + 1 < sFormatSpec.size()) && (sFormatSpec[i + 1] == '%')) + { + ++i; + } + } if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str()); // console + fputs(sLiteral.c_str(), stdout); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str()); // sample log file + fputs(sLiteral.c_str(), pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str()); // master log file + fputs(sLiteral.c_str(), pFileStream1); // master log file } } continue; @@ -420,14 +429,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list cArg = va_arg(vaArgList, char*); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), cArg); // console + printf(sFormatSpec.c_str(), cArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), cArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), cArg); // master log file } } continue; @@ -439,14 +448,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list iArg = va_arg(vaArgList, int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), iArg); // console + printf(sFormatSpec.c_str(), iArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), iArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), iArg); // master log file } } continue; @@ -460,14 +469,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list uiArg = va_arg(vaArgList, unsigned int); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), uiArg); // console + printf(sFormatSpec.c_str(), uiArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), uiArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), uiArg); // master log file } } continue; @@ -484,14 +493,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list dArg = va_arg(vaArgList, double); if (iLogMode & LOGCONSOLE) { - szNumWritten = printf(sFormatSpec.c_str(), dArg); // console + printf(sFormatSpec.c_str(), dArg); // console } if (iLogMode & LOGFILE) { - szNumWritten = fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file + fprintf (pFileStream0, sFormatSpec.c_str(), dArg); // sample log file if (iLogMode & MASTER) { - szNumWritten = fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file + fprintf(pFileStream1, sFormatSpec.c_str(), dArg); // master log file } } continue; @@ -501,14 +510,14 @@ static int shrLogV(int iLogMode, int iErrNum, const char* cFormatString, va_list // print arg of unknown/unsupported type to console and/or file if flagged if (iLogMode & LOGCONSOLE) // console { - szNumWritten = putc(*pStr, stdout); + putc(*pStr, stdout); } if (iLogMode & LOGFILE) { - szNumWritten = putc(*pStr, pFileStream0); // sample log file + putc(*pStr, pFileStream0); // sample log file if (iLogMode & MASTER) { - szNumWritten = putc(*pStr, pFileStream1); // master log file + putc(*pStr, pFileStream1); // master log file } } } diff --git a/tests/raytracing/rt_bvh_multinode/kernel.cpp b/tests/raytracing/rt_bvh_multinode/kernel.cpp index 0b31652e27..12eb9ff64a 100644 --- a/tests/raytracing/rt_bvh_multinode/kernel.cpp +++ b/tests/raytracing/rt_bvh_multinode/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { { r[3], r[4], r[5] }, r[6], r[7] }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_raycast/kernel.cpp b/tests/raytracing/rt_raycast/kernel.cpp index 9b8013146c..0ffc6baf7d 100644 --- a/tests/raytracing/rt_raycast/kernel.cpp +++ b/tests/raytracing/rt_raycast/kernel.cpp @@ -57,8 +57,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire the ray on the RTU (v2 single-issue trace). vx_ray_t ray = { {ro.x, ro.y, ro.z}, {rd.x, rd.y, rd.z}, 0.001f, 1e30f }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_replay/common.h b/tests/raytracing/rt_replay/common.h index 8f94077e7b..e696d13641 100644 --- a/tests/raytracing/rt_replay/common.h +++ b/tests/raytracing/rt_replay/common.h @@ -23,10 +23,9 @@ // One ray as the kernel consumes it. Each warp's rays share scene / flags / // cull: those ride the warp-uniform half of vx_rt_wtrace. typedef struct { - uint32_t scene; + uint64_t scene; uint32_t flags; uint32_t cull; - uint32_t pad; float origin[3]; float dir[3]; float tmin; @@ -57,7 +56,7 @@ typedef struct { #ifdef __cplusplus // Ray-log stream layout (mirror of sim/simx/rtu/rtu_raylog.h). #define RAYLOG_MAGIC 0x4C525856u -#define RAYLOG_VERSION 1u +#define RAYLOG_VERSION 2u #define RAYLOG_REC_LINE 1u #define RAYLOG_REC_RAY 2u #define RAYLOG_INFO_REPLAYABLE (1u << 8) @@ -69,11 +68,12 @@ struct RaylogHeader { struct RaylogRay { uint32_t type, info, seq, epoch; - uint32_t scene_root, ray_flags, cull_mask, warp_lane; + uint64_t scene_root; + uint32_t ray_flags, cull_mask, warp_lane; uint32_t origin[3], dir[3]; uint32_t tmin, tmax; uint32_t status, hit_t, hit_u, hit_v, prim, geom, inst_id, inst_custom, hit_attr; - uint32_t reserved[3]; + uint32_t reserved[2]; }; static_assert(sizeof(RaylogRay) == 112, "RaylogRay layout"); #endif diff --git a/tests/raytracing/rt_replay/kernel.cpp b/tests/raytracing/rt_replay/kernel.cpp index a6cf6922c5..24b6434c95 100644 --- a/tests/raytracing/rt_replay/kernel.cpp +++ b/tests/raytracing/rt_replay/kernel.cpp @@ -27,7 +27,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_ray_t ray = { {r.origin[0], r.origin[1], r.origin[2]}, {r.dir[0], r.dir[1], r.dir[2]}, r.tmin, r.tmax }; - uint32_t h = vx_rt_wtrace(r.scene, 0u, r.flags, r.cull, &ray); + uint32_t h = vx_rt_wtrace((uintptr_t)r.scene, 0u, r.flags, r.cull, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); uint32_t yielded = 0; diff --git a/tests/raytracing/rt_replay/main.cpp b/tests/raytracing/rt_replay/main.cpp index e326bcecfe..cf0dd113c7 100644 --- a/tests/raytracing/rt_replay/main.cpp +++ b/tests/raytracing/rt_replay/main.cpp @@ -164,9 +164,9 @@ bool matches(const RaylogRay& ref, const replay_result_t& got) { } void print_ray(FILE* fp, uint64_t idx, const RaylogRay& r, const replay_result_t& g, const char* cat) { - fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%x flags=0x%x cull=0x%x" + fprintf(fp, "%llu seq=%u cat=%s epoch=%u scene=0x%llx flags=0x%x cull=0x%x" " o=(%a,%a,%a) d=(%a,%a,%a) tmin=%a tmax=%a\n", - (unsigned long long)idx, r.seq, cat, r.epoch, r.scene_root, r.ray_flags, r.cull_mask, + (unsigned long long)idx, r.seq, cat, r.epoch, (unsigned long long)r.scene_root, r.ray_flags, r.cull_mask, fbits(r.origin[0]), fbits(r.origin[1]), fbits(r.origin[2]), fbits(r.dir[0]), fbits(r.dir[1]), fbits(r.dir[2]), fbits(r.tmin), fbits(r.tmax)); @@ -338,7 +338,7 @@ int main(int argc, char** argv) { // Group by the warp-uniform trace config; pad each group to whole warps // with copies of its last ray (their results are dropped). - std::map, std::vector> groups; + std::map, std::vector> groups; for (size_t k = pos; k < end; ++k) { const RaylogRay& r = rays[sel[k]]; groups[{r.scene_root, r.ray_flags & ~(VX_RT_FLAG_ENABLE_CHS | VX_RT_FLAG_ENABLE_MISS), r.cull_mask}].push_back(sel[k]); diff --git a/tests/raytracing/rt_smoke/kernel.cpp b/tests/raytracing/rt_smoke/kernel.cpp index 7271651877..dd265ad464 100644 --- a/tests/raytracing/rt_smoke/kernel.cpp +++ b/tests/raytracing/rt_smoke/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One trace + one wait. cull_mask = 0xff = no culling (the default). // payload = 0 (smoke test reads no payload). - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_ahs/kernel.cpp b/tests/raytracing/rt_smoke_ahs/kernel.cpp index 0019a747e0..0acb3685fc 100644 --- a/tests/raytracing/rt_smoke_ahs/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs/kernel.cpp @@ -44,8 +44,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait. Ray flags = 0 (no OPAQUE override; per-triangle flags // drive AHS). No payload. Any-hit candidates return to this warp; resume // with the decided action until a terminal status ends traversal. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp index 1953f25848..467439e216 100644 --- a/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_bvh/kernel.cpp @@ -43,8 +43,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Fire ray + wait for terminal. Ray flags = 0 (no OPAQUE override; // per-triangle flags drive AHS). No payload. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp index cfbe1c7e94..a476597782 100644 --- a/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_custom/kernel.cpp @@ -33,20 +33,20 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate instance attributes and the capture buffer pointer // (staged as the trace payload) from the register window, then stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_custom = vx_rt_get_attr(VX_RT_HIT_INSTANCE_CUSTOM, sts); uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_custom; // cand->cand_instance_custom cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp index cd33ff2f2b..d6bdadbe3c 100644 --- a/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_geom/kernel.cpp @@ -33,21 +33,21 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the non-opaque triangle drives the opacity classifier so // the walker yields an AHS candidate callback. The candidate-capture buffer - // rides the trace as the payload pointer (VX_RT_PAYLOAD_PTR_LO), so the + // rides the trace as the payload pointer (vx_rt_get_payload), so the // loop can locate it from the window. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t cand_lo = (uint32_t)(arg->cand_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, cand_lo, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t capture_ptr = arg->cand_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, capture_ptr, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate geometry/instance attributes and the capture buffer // pointer (staged as the trace payload) from the register window, stash. - uint32_t cand_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t cand_ptr = vx_rt_get_payload(sts); uint32_t cand_geom = vx_rt_get_attr(VX_RT_HIT_GEOMETRY_INDEX, sts) & VX_RT_HIT_GEOMETRY_MASK; uint32_t cand_inst = vx_rt_get_attr(VX_RT_HIT_INSTANCE_ID, sts); - uint32_t* cand = (uint32_t*)(uintptr_t)cand_ptr; + uint32_t* cand = (uint32_t*)cand_ptr; cand[0] = cand_geom; // cand->cand_geometry cand[1] = cand_inst; // cand->cand_instance_id sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); diff --git a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp index a5525d56b0..33ca412327 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_mixed/kernel.cpp @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmax, }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp index fdbe122e8d..96caf8da71 100644 --- a/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp +++ b/tests/raytracing/rt_smoke_ahs_multi/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { rtu_result_t* res = (rtu_result_t*)((uintptr_t)arg->results_addr) + tid; uint32_t n = 0; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_async_batch/kernel.cpp b/tests/raytracing/rt_smoke_async_batch/kernel.cpp index f7a0f9de1c..a9ce39b2db 100644 --- a/tests/raytracing/rt_smoke_async_batch/kernel.cpp +++ b/tests/raytracing/rt_smoke_async_batch/kernel.cpp @@ -41,10 +41,10 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Issue RTU_ASYNC_NUM_BATCH traces back-to-back. No WAIT in // between — each ray runs async in the cluster's RtuCore. - uint32_t h0 = vx_rt_wtrace((uint32_t)(arg->scene_addr[0] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h1 = vx_rt_wtrace((uint32_t)(arg->scene_addr[1] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h2 = vx_rt_wtrace((uint32_t)(arg->scene_addr[2] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); - uint32_t h3 = vx_rt_wtrace((uint32_t)(arg->scene_addr[3] & 0xffffffffu), 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h0 = vx_rt_wtrace(arg->scene_addr[0], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h1 = vx_rt_wtrace(arg->scene_addr[1], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h2 = vx_rt_wtrace(arg->scene_addr[2], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uint32_t h3 = vx_rt_wtrace(arg->scene_addr[3], 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); // Drain in declared order. Each WAIT blocks until its TERMINAL // delivers and applies the matching hit attrs. diff --git a/tests/raytracing/rt_smoke_bvh6/kernel.cpp b/tests/raytracing/rt_smoke_bvh6/kernel.cpp index 2b81b595bb..ef33e3ba4a 100644 --- a/tests/raytracing/rt_smoke_bvh6/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh6/kernel.cpp @@ -29,8 +29,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp index 1fecb230e1..c6a731bb9b 100644 --- a/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_basic/kernel.cpp @@ -26,8 +26,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp index 5dc276e80a..4344855937 100644 --- a/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_instanced/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp index 0154389871..00001a3025 100644 --- a/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp +++ b/tests/raytracing/rt_smoke_bvh_multilevel/kernel.cpp @@ -23,8 +23,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_chs/common.h b/tests/raytracing/rt_smoke_chs/common.h index e6abfdea09..d501295419 100644 --- a/tests/raytracing/rt_smoke_chs/common.h +++ b/tests/raytracing/rt_smoke_chs/common.h @@ -47,7 +47,7 @@ typedef struct { typedef struct { uint64_t scene_addr; uint64_t results_addr; - uint64_t payload_addr; // CHS reads VX_RT_PAYLOAD_PTR_LO/HI + uint64_t payload_addr; // CHS reads it back with vx_rt_get_payload uint32_t reserved; uint32_t reserved2; float ray_origin[3]; diff --git a/tests/raytracing/rt_smoke_chs/kernel.cpp b/tests/raytracing/rt_smoke_chs/kernel.cpp index c555e3f6a6..bd164b4bc2 100644 --- a/tests/raytracing/rt_smoke_chs/kernel.cpp +++ b/tests/raytracing/rt_smoke_chs/kernel.cpp @@ -37,17 +37,17 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the CHS loop body reads via // vx_gfx_get. Opt into CHS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the committed hit_t bits and the payload pointer from the window, // write MAGIC ^ hit_t, then resume: DONE drains the CHS-shaded hit. uint32_t hit_t_bits = vx_rt_get_attr(VX_RT_HIT_T, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_CHS_MAGIC ^ hit_t_bits; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_chs/main.cpp b/tests/raytracing/rt_smoke_chs/main.cpp index c5ee60896d..6e3b95c3ab 100644 --- a/tests/raytracing/rt_smoke_chs/main.cpp +++ b/tests/raytracing/rt_smoke_chs/main.cpp @@ -16,11 +16,13 @@ // Scene: one opaque triangle at z=5. Single lane fires a ray that // hits it. The kernel opts in to CHS via VX_RT_FLAG_ENABLE_CHS and // publishes a per-ray payload buffer pointer through -// VX_RT_PAYLOAD_PTR_LO. RtuCore commits the hit, then queues a +// VX_RT_PAYLOAD_PTR_LO/HI. RtuCore commits the hit, then queues a // CHS yield through the existing reformation/CB_YIELD path. The // dispatcher reads hit_t + payload pointer, writes MAGIC ^ hit_t_bits // to the payload, and exits via vx_rt_cb_ret(VX_RT_CB_DONE) + mret. // Oracle: status=HIT, payload[0] = MAGIC ^ bits(5.0f). +// On a 64-bit device the scene and the payload sit above 4 GiB, so the trace +// and the payload read-back must carry full-width pointers. #include #include @@ -52,6 +54,10 @@ vx_queue_h queue = nullptr; vx_module_h module_ = nullptr; vx_kernel_h kernel = nullptr; kernel_arg_t kernel_arg = {}; +bool high_mem = false; + +constexpr uint64_t HIGH_SCENE_ADDR = 0x140000000ull; +constexpr uint64_t HIGH_PAYLOAD_ADDR = 0x140010000ull; void cleanup() { if (device) { @@ -65,11 +71,23 @@ void cleanup() { } } +static vx_result_t create_buffer(uint64_t high_addr, uint64_t size, + uint32_t flags, vx_buffer_h* buf) { + if (high_mem) { + return vx_buffer_reserve(device, high_addr, size, flags, buf); + } + return vx_buffer_create(device, size, flags, buf); +} + int main(int /*argc*/, char* /*argv*/[]) { RT_CHECK(vx_device_open(0, &device)); vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; RT_CHECK(vx_queue_create(device, &qi, &queue)); + uint64_t isa_flags = 0; + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + high_mem = (VX_ISA_ARCH(isa_flags) == 64); + // Scene: 1 opaque triangle at z=5. std::vector scene_bytes(RTU_SCENE_HDR_BYTES + RTU_TRI_STRIDE_BYTES, 0); uint32_t* hdr = reinterpret_cast(scene_bytes.data()); @@ -83,7 +101,7 @@ int main(int /*argc*/, char* /*argv*/[]) { *tri_flags = RTU_TRI_FLAG_OPAQUE; uint32_t scene_bytes_sz = (uint32_t)scene_bytes.size(); - RT_CHECK(vx_buffer_create(device, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); + RT_CHECK(create_buffer(HIGH_SCENE_ADDR, scene_bytes_sz, VX_MEM_READ, &scene_buffer)); RT_CHECK(vx_buffer_address(scene_buffer, &kernel_arg.scene_addr)); uint32_t res_size = sizeof(rtu_result_t); @@ -93,8 +111,8 @@ int main(int /*argc*/, char* /*argv*/[]) { // Payload buffer: 4 bytes. Pre-init with a sentinel so we can tell // "CHS never ran" from "CHS ran and wrote 0". uint32_t payload_init = 0xdeadbeef; - RT_CHECK(vx_buffer_create(device, sizeof(uint32_t), - VX_MEM_READ_WRITE, &payload_buffer)); + RT_CHECK(create_buffer(HIGH_PAYLOAD_ADDR, sizeof(uint32_t), + VX_MEM_READ_WRITE, &payload_buffer)); RT_CHECK(vx_buffer_address(payload_buffer, &kernel_arg.payload_addr)); RT_CHECK(vx_enqueue_write(queue, payload_buffer, 0, &payload_init, sizeof(payload_init), 0, nullptr, nullptr)); diff --git a/tests/raytracing/rt_smoke_cull_back/kernel.cpp b/tests/raytracing/rt_smoke_cull_back/kernel.cpp index 25d4c99be3..492bccdf0a 100644 --- a/tests/raytracing/rt_smoke_cull_back/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_back/kernel.cpp @@ -33,8 +33,8 @@ static inline uint32_t fire_ray(uint64_t scene_addr, ray.dir[2] = dir[2]; ray.tmin = tmin; ray.tmax = tmax; - uint32_t scene_lo = (uint32_t)(scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_CULL_BACK_FACING, 0xffu, &ray); return vx_rt_wait(h, hit); diff --git a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp index 4e8ce8137b..38723d6ad5 100644 --- a/tests/raytracing/rt_smoke_cull_mask/kernel.cpp +++ b/tests/raytracing/rt_smoke_cull_mask/kernel.cpp @@ -36,12 +36,12 @@ __kernel void kernel_main(kernel_arg_t* arg) { ray.tmin = arg->tmin; ray.tmax = arg->tmax; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; rtu_result_t* results = (rtu_result_t*)((uintptr_t)arg->results_addr); // Each ray: fire+wait with a distinct cull mask, read results. for (uint32_t i = 0; i < RTU_NUM_RAYS; ++i) { - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, arg->ray_cull_mask[i], &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); results[tid].rays[i].status = sts; diff --git a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_deep_stack/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_stack/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp index ef23db8f9f..68c1941e68 100644 --- a/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_deep_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp index 3f7da83385..f83047fcd8 100644 --- a/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp +++ b/tests/raytracing/rt_smoke_fat_leaf/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp index b508ae0760..cc94b8342b 100644 --- a/tests/raytracing/rt_smoke_host_cfg/kernel.cpp +++ b/tests/raytracing/rt_smoke_host_cfg/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp index 2d078fc2cb..39ff1830e2 100644 --- a/tests/raytracing/rt_smoke_inst_flags/kernel.cpp +++ b/tests/raytracing/rt_smoke_inst_flags/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Ray flags = 0: the per-instance FORCE_NO_OPAQUE flag alone drives the // opacity classification, so the opaque triangle yields an AHS callback. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_is/kernel.cpp b/tests/raytracing/rt_smoke_is/kernel.cpp index 0d273775b1..32b56087bd 100644 --- a/tests/raytracing/rt_smoke_is/kernel.cpp +++ b/tests/raytracing/rt_smoke_is/kernel.cpp @@ -32,19 +32,19 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the candidate's callback type + payload pointer from the window. // Write MAGIC for a procedural candidate, a ~MAGIC sentinel otherwise. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t val = (cb_type == VX_RT_CB_TYPE_PROC) ? (uint32_t)RTU_IS_MAGIC : ~(uint32_t)RTU_IS_MAGIC; - *(uint32_t*)(uintptr_t)payload_ptr = val; + *(uint32_t*)payload_ptr = val; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_miss/kernel.cpp b/tests/raytracing/rt_smoke_miss/kernel.cpp index fba90e7b70..b120f863bf 100644 --- a/tests/raytracing/rt_smoke_miss/kernel.cpp +++ b/tests/raytracing/rt_smoke_miss/kernel.cpp @@ -35,15 +35,15 @@ __kernel void kernel_main(kernel_arg_t* arg) { // The trace stages the payload pointer the MISS loop body reads via // vx_gfx_get. Opt into MISS dispatch. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_MISS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { // Read the payload pointer from the window, write MAGIC, then resume. - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); - *(uint32_t*)(uintptr_t)payload_ptr = RTU_MISS_MAGIC; + uintptr_t payload_ptr = vx_rt_get_payload(sts); + *(uint32_t*)payload_ptr = RTU_MISS_MAGIC; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_multitri/kernel.cpp b/tests/raytracing/rt_smoke_multitri/kernel.cpp index 53b94f3172..bbc13fbff0 100644 --- a/tests/raytracing/rt_smoke_multitri/kernel.cpp +++ b/tests/raytracing/rt_smoke_multitri/kernel.cpp @@ -28,8 +28,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_proc/kernel.cpp b/tests/raytracing/rt_smoke_proc/kernel.cpp index 98e1c4a59d..17cf1d0967 100644 --- a/tests/raytracing/rt_smoke_proc/kernel.cpp +++ b/tests/raytracing/rt_smoke_proc/kernel.cpp @@ -35,9 +35,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // procedural primitive → IS decides (flags = 0), no payload. - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_recursive/kernel.cpp b/tests/raytracing/rt_smoke_recursive/kernel.cpp index 3cc63860c0..521d7496c1 100644 --- a/tests/raytracing/rt_smoke_recursive/kernel.cpp +++ b/tests/raytracing/rt_smoke_recursive/kernel.cpp @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { uint32_t tid = blockIdx.x; if (tid != 0) return; - uint32_t sub_scene = (uint32_t)(arg->sub_scene_addr & 0xffffffffu); + uintptr_t sub_scene = arg->sub_scene_addr; vx_ray_t ray = { { arg->ray_origin[0], arg->ray_origin[1], arg->ray_origin[2] }, @@ -37,11 +37,11 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); + uintptr_t scene_ptr = arg->scene_addr; // payload pointer for the CHS to write sub_status into; enable CHS for // the parent ray (so its closest hit returns a candidate). - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, VX_RT_FLAG_ENABLE_CHS, + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, VX_RT_FLAG_ENABLE_CHS, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); @@ -51,7 +51,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { vx_hit_t sub_hit; uint32_t sub_h = vx_rt_wtrace(sub_scene, 0u, 0u, 0xffu, &ray); uint32_t sub_status = vx_rt_wait(sub_h, &sub_hit); - *(volatile uint32_t*)(uintptr_t)payload = sub_status; + *(volatile uint32_t*)payload = sub_status; sts = vx_rt_continue(h, VX_RT_CB_DONE, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_reform/kernel.cpp b/tests/raytracing/rt_smoke_reform/kernel.cpp index fde753bffa..2339db5a09 100644 --- a/tests/raytracing/rt_smoke_reform/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform/kernel.cpp @@ -36,8 +36,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp index ed2fca2489..ff89e784d2 100644 --- a/tests/raytracing/rt_smoke_reform_mw/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_mw/kernel.cpp @@ -35,8 +35,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { diff --git a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp index f260332c21..a7a213c3ca 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_reform_sbt/kernel.cpp @@ -32,7 +32,7 @@ __kernel void kernel_main(kernel_arg_t* arg) { // One shared (warp-uniform) scene; the per-lane ray aims at tri `tid`, so // each lane gets a distinct sbt_idx from the tri it hits — divergence rides // the ray, not the scene pointer. - uint32_t scene_addr = (uint32_t)(arg->scene_base_addr & 0xffffffffu); + uintptr_t scene_addr = arg->scene_base_addr; float ox = (float)tid * RTU_TRI_SPACING + RTU_RAY_XOFF; vx_ray_t ray = { diff --git a/tests/raytracing/rt_smoke_sbt/kernel.cpp b/tests/raytracing/rt_smoke_sbt/kernel.cpp index d9b407b5f1..bcd4ad849c 100644 --- a/tests/raytracing/rt_smoke_sbt/kernel.cpp +++ b/tests/raytracing/rt_smoke_sbt/kernel.cpp @@ -46,9 +46,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { }; // The trace stages the payload pointer the SBT lookup reads via vx_gfx_get. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t payload = (uint32_t)(arg->payload_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, payload, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uintptr_t payload = arg->payload_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, payload, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); while (vx_rt_sts_is_yield(sts)) { @@ -56,9 +56,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // cb_type slot, load the selected magic, write it to the payload. uint32_t cb_type = vx_rt_get_attr(VX_RT_CB_TYPE, sts); uint32_t sbt_idx = vx_rt_get_attr(VX_RT_HIT_SBT_IDX, sts); - uint32_t payload_ptr = vx_rt_get_attr(VX_RT_PAYLOAD_PTR_LO, sts); + uintptr_t payload_ptr = vx_rt_get_payload(sts); uint32_t magic = sbt[sbt_idx * 4 + (cb_type - 1)]; - *(uint32_t*)(uintptr_t)payload_ptr = magic; + *(uint32_t*)payload_ptr = magic; sts = vx_rt_continue(h, VX_RT_CB_ACCEPT, hit.t, 0u, &hit); } diff --git a/tests/raytracing/rt_smoke_shadow/kernel.cpp b/tests/raytracing/rt_smoke_shadow/kernel.cpp index 6b5e4b21d0..5723306124 100644 --- a/tests/raytracing/rt_smoke_shadow/kernel.cpp +++ b/tests/raytracing/rt_smoke_shadow/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { arg->tmin, arg->tmax }; // Shadow-ray flags: opaque-force + terminate-on-first-hit. - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE | VX_RT_FLAG_TERMINATE_ON_FIRST_HIT, 0xffu, &ray); vx_hit_t hit; diff --git a/tests/raytracing/rt_smoke_tlas/kernel.cpp b/tests/raytracing/rt_smoke_tlas/kernel.cpp index 653139aa08..1f20dd3b2b 100644 --- a/tests/raytracing/rt_smoke_tlas/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, 0u, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, 0u, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp index d083e6b25d..833667843f 100644 --- a/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp +++ b/tests/raytracing/rt_smoke_tlas_builder/kernel.cpp @@ -25,8 +25,8 @@ __kernel void kernel_main(kernel_arg_t* arg) { {arg->ray_direction[0], arg->ray_direction[1], arg->ray_direction[2]}, arg->tmin, arg->tmax }; - uint32_t scene_lo = (uint32_t)(arg->scene_addr & 0xffffffffu); - uint32_t h = vx_rt_wtrace(scene_lo, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); + uintptr_t scene_ptr = arg->scene_addr; + uint32_t h = vx_rt_wtrace(scene_ptr, 0u, VX_RT_FLAG_OPAQUE, 0xffu, &ray); vx_hit_t hit; uint32_t sts = vx_rt_wait(h, &hit); diff --git a/tests/regression/async_barrier/kernel.cpp b/tests/regression/async_barrier/kernel.cpp index 8fc25a4b61..0a1b3aa082 100644 --- a/tests/regression/async_barrier/kernel.cpp +++ b/tests/regression/async_barrier/kernel.cpp @@ -24,11 +24,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto local_B0 = local_A0 + blockDim.x * blockDim.y; auto local_u32 = reinterpret_cast(local_B0 + blockDim.x * blockDim.y); - auto size = arg->size; - auto tile_size = arg->tile_size; + size_t size = arg->size; + size_t tile_size = arg->tile_size; - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; auto l_row = threadIdx.x; auto l_col = threadIdx.y; @@ -40,7 +40,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t token_compute; - for (uint32_t k = 0; k < size; k += tile_size) { + for (size_t k = 0; k < size; k += tile_size) { if (k > 0) { bar1.wait(token_compute); } @@ -52,7 +52,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE my_val = local_A0[l_row * tile_size + l_col]; TYPE my_val_b = local_B0[l_row * tile_size + l_col]; - for (uint32_t iter = 0; iter < tile_size; ++iter) { + for (size_t iter = 0; iter < tile_size; ++iter) { my_val = sqrt(my_val * my_val + 0.01f); my_val_b = sqrt(my_val_b * my_val_b + 0.01f); } @@ -60,7 +60,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { bar1.wait(token_load); - for (uint32_t j = 0; j < tile_size; ++j) { + for (size_t j = 0; j < tile_size; ++j) { sum += local_A0[l_row * tile_size + j] * local_B0[j * tile_size + l_col]; } token_compute = bar1.arrive(); diff --git a/tests/regression/basic/kernel.cpp b/tests/regression/basic/kernel.cpp index 5e8f51a124..31eb93f18d 100644 --- a/tests/regression/basic/kernel.cpp +++ b/tests/regression/basic/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/demo/kernel.cpp b/tests/regression/demo/kernel.cpp index 2199d70fa0..a4f5467bb4 100644 --- a/tests/regression/demo/kernel.cpp +++ b/tests/regression/demo/kernel.cpp @@ -11,11 +11,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // the flat tid = y*bdx+x reconstruction. uint32_t gx = blockIdx.x * blockDim.x + threadIdx.x; uint32_t gy = blockIdx.y * blockDim.y + threadIdx.y; - uint32_t gid = gy * arg->dim_x + gx; - uint32_t count = arg->task_size; - uint32_t offset = gid * count; + size_t gid = gy * arg->dim_x + gx; + size_t count = arg->task_size; + size_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } } diff --git a/tests/regression/dogfood/kernel.cpp b/tests/regression/dogfood/kernel.cpp index 889a063f50..a5ffa38a22 100644 --- a/tests/regression/dogfood/kernel.cpp +++ b/tests/regression/dogfood/kernel.cpp @@ -10,14 +10,14 @@ inline float __ieee754_sqrtf (float x) { } void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { int32_t a = src0_ptr[offset+i]; int32_t b = src1_ptr[offset+i]; int32_t c = a + b; @@ -26,14 +26,14 @@ void kernel_iadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -42,14 +42,14 @@ void kernel_imul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -58,14 +58,14 @@ void kernel_idiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -76,14 +76,14 @@ void kernel_idiv_mul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { float a = src0_ptr[offset+i]; float b = src1_ptr[offset+i]; float c = a + b; @@ -92,14 +92,14 @@ void kernel_fadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a - b; @@ -108,14 +108,14 @@ void kernel_fsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b; @@ -124,14 +124,14 @@ void kernel_fmul(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b + b; @@ -140,14 +140,14 @@ void kernel_fmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a * b - b; @@ -156,14 +156,14 @@ void kernel_fmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -172,14 +172,14 @@ void kernel_fnmadd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b + b; @@ -188,14 +188,14 @@ void kernel_fnmsub(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c =-a * b - b; @@ -206,14 +206,14 @@ void kernel_fnmadd_madd(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -222,14 +222,14 @@ void kernel_fdiv(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a / b; @@ -240,14 +240,14 @@ void kernel_fdiv2(kernel_arg_t* __UNIFORM__ arg) { } void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = __ieee754_sqrtf(a * b); @@ -256,14 +256,14 @@ void kernel_fsqrt(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (int32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -273,14 +273,14 @@ void kernel_ftoi(kernel_arg_t* __UNIFORM__ arg) { } void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (uint32_t*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -290,14 +290,14 @@ void kernel_ftou(kernel_arg_t* __UNIFORM__ arg) { } void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -307,14 +307,14 @@ void kernel_itof(kernel_arg_t* __UNIFORM__ arg) { } void kernel_utof(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int32_t*)arg->src0_addr; auto src1_ptr = (int32_t*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; auto c = a + b; @@ -328,14 +328,14 @@ inline float fclamp(float a, float b, float c) { } void kernel_fclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = fclamp(1.0f, a, b); @@ -347,14 +347,14 @@ inline int iclamp(int a, int b, int c) { } void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (int*)arg->src0_addr; auto src1_ptr = (int*)arg->src1_addr; auto dst_ptr = (int*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { auto a = src0_ptr[offset+i]; auto b = src1_ptr[offset+i]; dst_ptr[offset+i] = iclamp(1, a, b); @@ -362,14 +362,14 @@ void kernel_iclamp(kernel_arg_t* __UNIFORM__ arg) { } void kernel_trigo(kernel_arg_t* __UNIFORM__ arg) { - auto count = arg->task_size; + size_t count = arg->task_size; auto src0_ptr = (float*)arg->src0_addr; auto src1_ptr = (float*)arg->src1_addr; auto dst_ptr = (float*)arg->dst_addr; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; auto offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { - uint32_t j = offset + i; + for (size_t i = 0; i < count; ++i) { + size_t j = offset + i; auto a = src0_ptr[j]; auto b = src1_ptr[j]; auto c = a * b; diff --git a/tests/regression/dogfood/testcases.h b/tests/regression/dogfood/testcases.h index f3562bb179..29e8e819c1 100644 --- a/tests/regression/dogfood/testcases.h +++ b/tests/regression/dogfood/testcases.h @@ -351,7 +351,7 @@ class Test_FMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] + b[i]; + auto ref = fmaf(a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -381,7 +381,7 @@ class Test_FMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = a[i] * b[i] - b[i]; + auto ref = fmaf(a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -411,7 +411,7 @@ class Test_FNMADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] - b[i]; + auto ref = fmaf(-a[i], b[i], -b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -441,7 +441,7 @@ class Test_FNMSUB : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto ref = -a[i] * b[i] + b[i]; + auto ref = fmaf(-a[i], b[i], b[i]); if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; ++errors; @@ -471,8 +471,8 @@ class Test_FNMADD_MADD : public ITestCase { auto b = (float*)src2; auto c = (float*)dst; for (uint32_t i = 0; i < n; ++i) { - auto x = -a[i] * b[i] - b[i]; - auto y = a[i] * b[i] + b[i]; + auto x = fmaf(-a[i], b[i], -b[i]); + auto y = fmaf(a[i], b[i], b[i]); auto ref = x + y; if (!almost_equal(c[i], ref)) { std::cout << "error at result #" << std::dec << i << std::hex << ": expected=" << ref << ", actual=" << c[i] << ", a=" << a[i] << ", b=" << b[i] << std::endl; diff --git a/tests/regression/dotproduct2/kernel.cpp b/tests/regression/dotproduct2/kernel.cpp index c10a20c2bf..bf7ead6823 100644 --- a/tests/regression/dotproduct2/kernel.cpp +++ b/tests/regression/dotproduct2/kernel.cpp @@ -5,8 +5,8 @@ #define VX_CFG_NUM_THREADS 4 #endif -static inline size_t bits_from_f32(float x) { size_t u; __builtin_memcpy(&u, &x, 4); return u; } -static inline float f32_from_bits(size_t u) { float x; __builtin_memcpy(&x, &u, 4); return x; } +static inline size_t bits_from_f32(float x) { uint32_t u; __builtin_memcpy(&u, &x, 4); return u; } +static inline float f32_from_bits(size_t u) { uint32_t b = u; float x; __builtin_memcpy(&x, &b, 4); return x; } static inline float warp_reduce_sum(float x, uint32_t warp_size) { int clamp = warp_size - 1; diff --git a/tests/regression/dropout/kernel.cpp b/tests/regression/dropout/kernel.cpp index 95e77d8189..c9cd35b5fe 100644 --- a/tests/regression/dropout/kernel.cpp +++ b/tests/regression/dropout/kernel.cpp @@ -10,6 +10,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto multiplier = arg->multiplier; uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } float rand_value = RandomFloat(WangHash(gid)); TYPE scaled_value = src0_ptr[gid] * multiplier; dst_ptr[gid] = (rand_value < dropout_p) ? 0.0 : scaled_value; diff --git a/tests/regression/dxa_copy_mcast/kernel.cpp b/tests/regression/dxa_copy_mcast/kernel.cpp index d56b2558c5..c25a792719 100644 --- a/tests/regression/dxa_copy_mcast/kernel.cpp +++ b/tests/regression/dxa_copy_mcast/kernel.cpp @@ -14,9 +14,9 @@ constexpr uint32_t kDescSrc = 0; __kernel void kernel_main(kernel_arg_t* arg) { - const uint32_t cta_id = get_local_group_id(); - const uint32_t tile_rows = arg->tile_rows; - const uint32_t tile_cols = arg->tile_cols; + const size_t cta_id = get_local_group_id(); + const size_t tile_rows = arg->tile_rows; + const size_t tile_cols = arg->tile_cols; const uint32_t num_recv = arg->num_recv; auto shmem = reinterpret_cast(__local_mem()); @@ -42,9 +42,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { // Store the received tile to dst region for host verification. // Layout: dst[cta_id * tile_elems + r * tile_cols + c] auto dst = reinterpret_cast(arg->dst_addr); - const uint32_t tile_elems = tile_rows * tile_cols; - const uint32_t l_col = threadIdx.x; - for (uint32_t r = 0; r < tile_rows; ++r) { + const size_t tile_elems = tile_rows * tile_cols; + const size_t l_col = threadIdx.x; + for (size_t r = 0; r < tile_rows; ++r) { if (l_col < tile_cols) { dst[cta_id * tile_elems + r * tile_cols + l_col] = shmem[r * tile_cols + l_col]; diff --git a/tests/regression/dxa_kmajor_check/kernel.cpp b/tests/regression/dxa_kmajor_check/kernel.cpp index de381140da..d1bcc86f8c 100644 --- a/tests/regression/dxa_kmajor_check/kernel.cpp +++ b/tests/regression/dxa_kmajor_check/kernel.cpp @@ -14,7 +14,7 @@ constexpr uint32_t kDescA = 0; constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - const uint32_t tid = threadIdx.x; + const size_t tid = threadIdx.x; const uint32_t cta_id = blockIdx.x; // 1D grid — one CTA per (M-tile, N-tile) pair const uint32_t grid_n = arg->N / arg->tileN; const uint32_t cta_row = cta_id / grid_n; @@ -37,8 +37,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // Copy both SMEM regions to dst[cta_id * cta_bytes ..] auto pDst = reinterpret_cast(arg->dst_addr) + cta_id * arg->cta_bytes; - const uint32_t total = arg->cta_bytes; - for (uint32_t i = tid; i < total; i += blockDim.x) { + const size_t total = arg->cta_bytes; + for (size_t i = tid; i < total; i += blockDim.x) { pDst[i] = A_smem[i]; // A_smem is followed by B_smem in memory } __syncthreads(); diff --git a/tests/regression/fence/kernel.cpp b/tests/regression/fence/kernel.cpp index 8d0c8576ea..1e7a6ba47f 100644 --- a/tests/regression/fence/kernel.cpp +++ b/tests/regression/fence/kernel.cpp @@ -2,14 +2,14 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t count = arg->task_size; + size_t count = arg->task_size; int32_t* src0_ptr = (int32_t*)arg->src0_addr; int32_t* src1_ptr = (int32_t*)arg->src1_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * count; - for (uint32_t i = 0; i < count; ++i) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * count; + for (size_t i = 0; i < count; ++i) { dst_ptr[offset+i] = src0_ptr[offset+i] + src1_ptr[offset+i]; } diff --git a/tests/regression/gfx_binsort_kernel/kernel.cpp b/tests/regression/gfx_binsort_kernel/kernel.cpp index 53e11bdc03..0791bdb204 100644 --- a/tests/regression/gfx_binsort_kernel/kernel.cpp +++ b/tests/regression/gfx_binsort_kernel/kernel.cpp @@ -34,9 +34,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto meta = reinterpret_cast(arg->meta_addr); const uint32_t n = arg->num_prims; - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = BINSORT_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = BINSORT_NUM_BINS; switch (arg->stage) { @@ -69,7 +69,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); if (tid == 0) { uint32_t acc = 0; - for (uint32_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } + for (size_t t = 0; t < T; ++t) { uint32_t v = tsum[t]; tsum[t] = acc; acc += v; } meta[0] = acc; offset[n] = acc; } __syncthreads(); @@ -82,7 +82,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (uint32_t i = lo; i < hi; ++i) { if (count[i] == 0) continue; // culled / empty bbox int bL, bR, bT, bB; bin_range(prims[i], bL, bR, bT, bB); - uint32_t w = offset[i]; + size_t w = offset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * BINSORT_BIN_COLS + bx) << BINSORT_PRIM_BITS) | i; @@ -98,7 +98,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> BINSORT_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case BINSORT_STAGE_BASE: { // single CTA: bin-base scan + headers @@ -123,7 +123,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t kchunk = (P + T - 1) / T; uint32_t klo = umin(tid * kchunk, P), khi = umin(klo + kchunk, P); // owned hist columns -> stable write cursors (base + thread-exclusive prefix) - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> BINSORT_PRIM_BITS; diff --git a/tests/regression/gfx_pipeline_kernel/kernel.cpp b/tests/regression/gfx_pipeline_kernel/kernel.cpp index 765cfc8b16..1d17eb3703 100644 --- a/tests/regression/gfx_pipeline_kernel/kernel.cpp +++ b/tests/regression/gfx_pipeline_kernel/kernel.cpp @@ -62,9 +62,9 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case PIPE_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { rast_prim_t pr[SETUP_MAX_SUB]; setup_bbox_t bb[SETUP_MAX_SUB]; uint32_t kept = clip_and_setup(&verts[3 * t], W, H, pr, bb); @@ -91,12 +91,12 @@ __kernel void setup_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < ntri; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < ntri; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; prim[w] = slot_prim[slot]; bbox[w] = slot_bbox[slot]; ++w; @@ -120,17 +120,17 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto tilebuf = reinterpret_cast(arg->tilebuf_addr); auto meta = reinterpret_cast(arg->meta_addr); - const uint32_t tid = threadIdx.x; - const uint32_t T = blockDim.x; - const uint32_t B = PIPE_NUM_BINS; + const size_t tid = threadIdx.x; + const size_t T = blockDim.x; + const size_t B = PIPE_NUM_BINS; switch (arg->stage) { case PIPE_STAGE_BCOUNT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); bcount[i] = (uint32_t)((bR - bL + 1) * (bB - bT + 1)); } @@ -152,12 +152,12 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { } break; case PIPE_STAGE_BEMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; uint32_t P = meta[0]; - for (uint32_t i = gid; i < P; i += gstride) { + for (size_t i = gid; i < P; i += gstride) { int bL, bR, bT, bB; bin_range(bbox[i], bL, bR, bT, bB); - uint32_t w = boffset[i]; + size_t w = boffset[i]; for (int by = bT; by <= bB; ++by) for (int bx = bL; bx <= bR; ++bx) keys[w++] = ((uint32_t)(by * PIPE_BIN_COLS + bx) << PIPE_PRIM_BITS) | i; @@ -173,7 +173,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t b = keys[k] >> PIPE_PRIM_BITS; if (b >= lo && b < hi) thist[tid * B + b]++; } __syncthreads(); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (uint32_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t s = 0; for (size_t t = 0; t < T; ++t) s += thist[t * B + b]; bincount[b] = s; } } break; case PIPE_STAGE_BBASE: { // single CTA: tilebuf headers @@ -205,7 +205,7 @@ __kernel void binning_k(kernel_arg_t* __UNIFORM__ arg) { auto pids = reinterpret_cast(tilebuf + nb * sizeof(rast_tile_header_t)); uint32_t kchunk = (K + T - 1) / T; uint32_t klo = umin(tid * kchunk, K), khi = umin(klo + kchunk, K); - for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (uint32_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } + for (uint32_t b = lo + tid; b < hi; b += T) { uint32_t run = binbase[b]; for (size_t t = 0; t < T; ++t) { uint32_t c = thist[t * B + b]; thist[t * B + b] = run; run += c; } } __syncthreads(); for (uint32_t k = klo; k < khi; ++k) { uint32_t key = keys[k], b = key >> PIPE_PRIM_BITS; diff --git a/tests/regression/gfx_setup_kernel/kernel.cpp b/tests/regression/gfx_setup_kernel/kernel.cpp index 657311e37a..d20c43dd83 100644 --- a/tests/regression/gfx_setup_kernel/kernel.cpp +++ b/tests/regression/gfx_setup_kernel/kernel.cpp @@ -38,9 +38,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { switch (arg->stage) { case SETUP_STAGE_SETUP: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { clip_tri_t sub[SETUP_MAX_SUB]; int ns = clip_near(verts[3 * t + 0], verts[3 * t + 1], verts[3 * t + 2], sub); uint32_t kept = 0; @@ -49,7 +49,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { setup_bbox_t bbox{}; if (setup_triangle(sub[s].v[0], sub[s].v[1], sub[s].v[2], W, H, SETUP_NEAR, SETUP_FAR, prim, bbox, arg->cull_mode)) { - uint32_t slot = t * SETUP_MAX_SUB + kept; + size_t slot = t * SETUP_MAX_SUB + kept; slot_prim[slot] = prim; slot_bbox[slot] = bbox; slot_vtx[slot] = sub[s]; @@ -75,12 +75,12 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } break; case SETUP_STAGE_EMIT: { // multi-CTA, grid-stride - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t gstride = gridDim.x * blockDim.x; - for (uint32_t t = gid; t < n; t += gstride) { - uint32_t w = offset[t]; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t gstride = gridDim.x * blockDim.x; + for (size_t t = gid; t < n; t += gstride) { + size_t w = offset[t]; for (uint32_t s = 0; s < keep[t]; ++s) { - uint32_t slot = t * SETUP_MAX_SUB + s; + size_t slot = t * SETUP_MAX_SUB + s; out_prim[w] = slot_prim[slot]; out_bbox[w] = slot_bbox[slot]; out_vtx[w] = slot_vtx[slot]; diff --git a/tests/regression/jacobi/kernel.cpp b/tests/regression/jacobi/kernel.cpp index 5b16c63cf9..9e10782d50 100644 --- a/tests/regression/jacobi/kernel.cpp +++ b/tests/regression/jacobi/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto x_old = reinterpret_cast(arg->src1_addr); auto b = reinterpret_cast(arg->src2_addr); auto x_new = reinterpret_cast(arg->dst_addr); - auto n = arg->size; + size_t n = arg->size; - uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + size_t i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; - uint32_t index = i * n; + size_t index = i * n; double sum = 0.0; - for (uint32_t j = 0; j < n; j++) { + for (size_t j = 0; j < n; j++) { if (j != i) sum += A[index + j] * x_old[j]; } diff --git a/tests/regression/minimal/kernel.cpp b/tests/regression/minimal/kernel.cpp index 0d04cd2563..9cdf8cf52b 100644 --- a/tests/regression/minimal/kernel.cpp +++ b/tests/regression/minimal/kernel.cpp @@ -7,5 +7,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = MINIMAL_MAGIC | idx; } diff --git a/tests/regression/module_reload/kernel.cpp b/tests/regression/module_reload/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/module_reload/kernel.cpp +++ b/tests/regression/module_reload/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index f3b5d2d212..8a66f2047c 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -2,18 +2,18 @@ #include "common.h" __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - uint32_t stride = arg->stride; + size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; float* src_ptr = (float*)arg->src1_addr; float* dst_ptr = (float*)arg->dst_addr; - uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t offset = gid * stride; + size_t gid = blockIdx.x * blockDim.x + threadIdx.x; + size_t offset = gid * stride; - for (uint32_t i = 0; i < stride; ++i) { + for (size_t i = 0; i < stride; ++i) { float value = 0.0f; for (uint32_t j = 0; j < NUM_LOADS; ++j) { - uint32_t addr = offset + i + j; + size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; value *= src_ptr[index]; } diff --git a/tests/regression/packld/kernel.cpp b/tests/regression/packld/kernel.cpp index 0d3bef2215..5473c06d22 100644 --- a/tests/regression/packld/kernel.cpp +++ b/tests/regression/packld/kernel.cpp @@ -14,7 +14,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; uint32_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { diff --git a/tests/regression/packld_box/kernel.cpp b/tests/regression/packld_box/kernel.cpp index b407b12518..e41ef52e7f 100644 --- a/tests/regression/packld_box/kernel.cpp +++ b/tests/regression/packld_box/kernel.cpp @@ -15,7 +15,7 @@ // Register spilling silently erases the bug by doing proper NaN-boxing. // Uncomment the two lines to see all the tests pass. __attribute__((always_inline)) -inline uint64_t packlb_bits(const void* base, uint32_t stride) { +inline uint64_t packlb_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 1, 4, ft0, %2, %3\n\t" @@ -28,7 +28,7 @@ inline uint64_t packlb_bits(const void* base, uint32_t stride) { } __attribute__((always_inline)) -inline uint64_t packlh_bits(const void* base, uint32_t stride) { +inline uint64_t packlh_bits(const void* base, size_t stride) { uint64_t bits; __asm__ volatile ( ".insn r %1, 2, 4, ft0, %2, %3\n\t" @@ -53,8 +53,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_lb = reinterpret_cast(arg->dst_lb_addr); auto dst_lh = reinterpret_cast(arg->dst_lh_addr); - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t stride = 1; // byte stride between consecutive elements + size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + size_t stride = 1; // byte stride between consecutive elements for (uint32_t p = 0; p < NUM_POINTS; ++p) { // 4 bytes at consecutive addresses → one packed float (PACKLB) diff --git a/tests/regression/raycast/render.h b/tests/regression/raycast/render.h index f793a6e946..b75d87fc84 100644 --- a/tests/regression/raycast/render.h +++ b/tests/regression/raycast/render.h @@ -78,7 +78,7 @@ void BVHIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = 0; // Push root node index while (stackPtr != 0) { @@ -86,7 +86,7 @@ void BVHIntersect(const ray_t &ray, const bvh_node_t &node = bvhBuffer[nodeIdx]; if (node.isLeaf()) { // Intersect leaf triangles - for (uint32_t i = 0; i < node.triCount; ++i) { + for (size_t i = 0; i < node.triCount; ++i) { uint32_t triIdx = triIdxBuffer[node.leftFirst + i]; float dist; float3_t bcoords; @@ -149,7 +149,7 @@ void TLASIntersect(const ray_t &ray, const tri_t *triBuffer, ray_hit_t *hit) { uint32_t stack[BVH_STACK_SIZE]; - uint32_t stackPtr = 0; + size_t stackPtr = 0; stack[stackPtr++] = tlas_root; while (stackPtr != 0) { diff --git a/tests/regression/relu/kernel.cpp b/tests/regression/relu/kernel.cpp index 55335af652..b70730b79a 100644 --- a/tests/regression/relu/kernel.cpp +++ b/tests/regression/relu/kernel.cpp @@ -6,6 +6,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= arg->num_points) { + return; + } TYPE value = src0_ptr[gid]; dst_ptr[gid] = (value < 0) ? TYPE(0) : value; } diff --git a/tests/regression/reopen/kernel.cpp b/tests/regression/reopen/kernel.cpp index 1ebcfeddae..0491f994d1 100644 --- a/tests/regression/reopen/kernel.cpp +++ b/tests/regression/reopen/kernel.cpp @@ -3,13 +3,13 @@ int main() { kernel_arg_t* __UNIFORM__ arg = (kernel_arg_t*)csr_read(VX_CSR_MSCRATCH); - uint32_t count = arg->count; + size_t count = arg->count; int32_t* src_ptr = (int32_t*)arg->src_addr; int32_t* dst_ptr = (int32_t*)arg->dst_addr; - uint32_t offset = vx_core_id() * count; + size_t offset = vx_core_id() * count; - for (uint32_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { dst_ptr[offset + i] = src_ptr[offset + i]; } diff --git a/tests/regression/sgemm/kernel.cpp b/tests/regression/sgemm/kernel.cpp index bafb73cb91..7b79bbfb11 100644 --- a/tests/regression/sgemm/kernel.cpp +++ b/tests/regression/sgemm/kernel.cpp @@ -5,14 +5,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; TYPE sum(0); #pragma unroll 8 - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index 17a8817f9a..afe6c336e8 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -6,15 +6,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -26,19 +26,19 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_dxa/kernel.cpp b/tests/regression/sgemm2_dxa/kernel.cpp index f152f52275..7693034844 100644 --- a/tests/regression/sgemm2_dxa/kernel.cpp +++ b/tests/regression/sgemm2_dxa/kernel.cpp @@ -11,9 +11,9 @@ constexpr uint32_t kDescB = 1; // Accumulate partial sum from shared memory tiles. // shA layout: [tile_size × chunk_k], row-major (row = l_row, col = k) // shB layout: [chunk_k × tile_size], row-major (row = k, col = l_col) -static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, uint32_t tile_size, - uint32_t chunk_k, uint32_t l_row, uint32_t l_col) { - for (uint32_t k = 0; k < chunk_k; ++k) { +static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, size_t tile_size, + size_t chunk_k, size_t l_row, size_t l_col) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; } } @@ -21,25 +21,25 @@ static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mode = arg->mode; // Global and local thread coordinates. - const uint32_t row_base = blockIdx.x * tile_size; - const uint32_t col_base = blockIdx.y * tile_size; - const uint32_t l_row = threadIdx.x; - const uint32_t l_col = threadIdx.y; - const uint32_t g_row = row_base + l_row; - const uint32_t g_col = col_base + l_col; + const size_t row_base = blockIdx.x * tile_size; + const size_t col_base = blockIdx.y * tile_size; + const size_t l_row = threadIdx.x; + const size_t l_col = threadIdx.y; + const size_t g_row = row_base + l_row; + const size_t g_col = col_base + l_col; TYPE sum(0); // SMEM tile layout: A tile is [tile_size x chunk_k], B tile is [chunk_k x tile_size]. - const uint32_t tile_elems_a = tile_size * chunk_k; - const uint32_t tile_elems_b = chunk_k * tile_size; - const uint32_t stage_elems = tile_elems_a + tile_elems_b; + const size_t tile_elems_a = tile_size * chunk_k; + const size_t tile_elems_b = chunk_k * tile_size; + const size_t stage_elems = tile_elems_a + tile_elems_b; // const uint32_t stage_count = (mode == 2) ? 2u : 1u; // Allocate shared memory for tile buffers. diff --git a/tests/regression/sgemm2_dxa_mcast/kernel.cpp b/tests/regression/sgemm2_dxa_mcast/kernel.cpp index 1127865e94..7552459835 100644 --- a/tests/regression/sgemm2_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm2_dxa_mcast/kernel.cpp @@ -26,16 +26,16 @@ constexpr uint32_t kDescB = 1; __kernel void kernel_main(kernel_arg_t* arg) { auto C = reinterpret_cast(arg->C_addr); - const uint32_t size = arg->size; - const uint32_t tile_size = arg->tile_size; - const uint32_t chunk_k = arg->chunk_k; + const size_t size = arg->size; + const size_t tile_size = arg->tile_size; + const size_t chunk_k = arg->chunk_k; const uint32_t mc_group_size = arg->mc_group_size; // CTA position. blockIdx.x walks rows; blockIdx.y walks column-blocks. // 4 CTAs (mc_group_size) in adjacent blockIdx.x values are co-resident // (= one local group) and share one B column-block via multicast. - const uint32_t row = blockIdx.x; - const uint32_t col_blk = blockIdx.y; + const size_t row = blockIdx.x; + const size_t col_blk = blockIdx.y; // SMEM tile layout: A row [chunk_k] then B column-block [chunk_k × tile_size]. auto shmem = reinterpret_cast(__local_mem()); @@ -68,9 +68,9 @@ __kernel void kernel_main(kernel_arg_t* arg) { local_B.arrive_and_wait(); // ── Compute: 1 row × tile_size cols ─────────────────────────────────────── - const uint32_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) + const size_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) TYPE sum(0); - for (uint32_t k = 0; k < chunk_k; ++k) { + for (size_t k = 0; k < chunk_k; ++k) { sum += shA[k] * shB[k * tile_size + l_col]; } diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index ff7c717dd0..3cc808bf0d 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t cta_M = arg->cta_M; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; // num_warps * VX_CFG_NUM_THREADS uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::tileN; // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] row-major auto smem = reinterpret_cast(__local_mem()); @@ -32,23 +32,23 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } // Cooperatively load B tile [tileK x tileN] from DRAM (col-major source) // and store row-major in shared memory. uint32_t b_size = ctx::tileK * ctx::tileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileN; - uint32_t c = idx % ctx::tileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileN; + size_t c = idx % ctx::tileN; B_smem[r * ctx::tileN + c] = pB[(tile_col + c) * K + (k + r)]; } diff --git a/tests/regression/sgemm2_v1/kernel.cpp b/tests/regression/sgemm2_v1/kernel.cpp index e26687a322..9f32b87f3d 100644 --- a/tests/regression/sgemm2_v1/kernel.cpp +++ b/tests/regression/sgemm2_v1/kernel.cpp @@ -6,15 +6,15 @@ void kernel_body(kernel_arg_t *arg) { auto B_ptr = reinterpret_cast(arg->B_addr); auto C_ptr = reinterpret_cast(arg->C_addr); - auto size = arg->size; - auto tile_size = arg->tile_size; - auto chunk_k = arg->chunk_k; + size_t size = arg->size; + size_t tile_size = arg->tile_size; + size_t chunk_k = arg->chunk_k; // Global and local thread coordinates. - auto g_row = blockIdx.x * blockDim.x + threadIdx.x; - auto g_col = blockIdx.y * blockDim.y + threadIdx.y; - auto l_row = threadIdx.x; - auto l_col = threadIdx.y; + size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; + size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; + size_t l_row = threadIdx.x; + size_t l_col = threadIdx.y; // Shared memory layout: // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) @@ -27,19 +27,19 @@ void kernel_body(kernel_arg_t *arg) { TYPE sum(0); - for (uint32_t k = 0; k < size; k += chunk_k) { + for (size_t k = 0; k < size; k += chunk_k) { // Cooperative load: each thread loads chunk_k/tile_size elements per tile. // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (uint32_t kk = l_col; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; } - for (uint32_t kk = l_row; kk < chunk_k; kk += tile_size) { + for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; } __syncthreads(); - for (uint32_t j = 0; j < chunk_k; ++j) { + for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } diff --git a/tests/regression/sgemm_tcu/kernel.cpp b/tests/regression/sgemm_tcu/kernel.cpp index 32cfd94d35..b3f8c8d2d9 100644 --- a/tests/regression/sgemm_tcu/kernel.cpp +++ b/tests/regression/sgemm_tcu/kernel.cpp @@ -11,16 +11,16 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pB = reinterpret_cast(arg->B_addr); auto pC = reinterpret_cast(arg->C_addr); - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; ctx::fragment_a fragA; ctx::fragment_b fragB; ctx::fragment_acc fragC; // calculate tile row & column based on block index - uint32_t tile_row = blockIdx.y * ctx::tileM; - uint32_t tile_col = blockIdx.x * ctx::tileN; + size_t tile_row = blockIdx.y * ctx::tileM; + size_t tile_col = blockIdx.x * ctx::tileN; // Initialize accumulator tile to zero ctx::fill_fragment(fragC, 0); @@ -29,7 +29,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t cycles = 0; #endif - for (uint32_t i = 0; i < K; i += ctx::tileK) { + for (size_t i = 0; i < K; i += ctx::tileK) { auto pTileA = pA + tile_row * K + i; auto pTileB = pB + tile_col * K + i; diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 05dfd78059..7b2596dabb 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -75,7 +75,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); // Only the first warp in the CTA issues DXA commands. - const bool is_dxa_warp = (get_sub_group_id() == 0); + [[maybe_unused]] const bool is_dxa_warp = (get_sub_group_id() == 0); // Stage the tile at K-offset k into the {A_smem, B_smem} buffer guarded // by barrier id `bar`. All state is passed by value in registers. @@ -99,20 +99,20 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #ifdef SW_LOAD_A // Cooperative load A row-major (matches DXA row-major layout). uint32_t a_size = cta_M * ctx::tileK; - for (uint32_t i = 0; i < a_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::tileK; - uint32_t c = idx % ctx::tileK; + for (size_t i = 0; i < a_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::tileK; + size_t c = idx % ctx::tileK; A_smem[r * ctx::tileK + c] = pA[(tile_row + r) * K + (k + c)]; } #endif #ifdef SW_LOAD_B // Cooperative load B block-major (matches DXA BlockMajor LAYOUT / bbuf). uint32_t b_size = ctx::tileK * ctx::xtileN; - for (uint32_t i = 0; i < b_size; i += num_threads) { - uint32_t idx = i + tid; - uint32_t r = idx / ctx::xtileN; - uint32_t c = idx % ctx::xtileN; + for (size_t i = 0; i < b_size; i += num_threads) { + size_t idx = i + tid; + size_t r = idx / ctx::xtileN; + size_t c = idx % ctx::xtileN; B_smem[ctx::b_blockmajor_idx(r, c)] = pB[(k + r) * N + (tile_col + c)]; } #endif diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index f370bbd583..34bb9e5c9c 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -19,15 +19,15 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pBTensorScale = reinterpret_cast(arg->B_tensor_scale_addr); #endif - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t num_threads = blockDim.x; uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; uint32_t num_warps = num_threads / VX_CFG_NUM_THREADS; uint32_t cta_m = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_m; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_m; + size_t tile_col = blockIdx.x * ctx::xtileN; uint32_t num_k_tiles = K / ctx::tileK; auto A_smem = reinterpret_cast(__local_mem()); @@ -37,18 +37,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; - for (uint32_t idx = tid; idx < a_bytes; idx += num_threads) { - uint32_t row = idx / ctx::tileK; - uint32_t col = idx % ctx::tileK; + for (size_t idx = tid; idx < a_bytes; idx += num_threads) { + size_t row = idx / ctx::tileK; + size_t col = idx % ctx::tileK; A_smem[idx] = pA[(tile_row + row) * K + k + col]; } uint32_t b_bytes = ctx::xtileN * ctx::tileK; - for (uint32_t idx = tid; idx < b_bytes; idx += num_threads) { - uint32_t col = idx / ctx::tileK; - uint32_t k_byte = idx % ctx::tileK; + for (size_t idx = tid; idx < b_bytes; idx += num_threads) { + size_t col = idx / ctx::tileK; + size_t k_byte = idx % ctx::tileK; B_smem[idx] = pB[(tile_col + col) * K + k + k_byte]; } __syncthreads(); diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index 8c60045c2e..f1872cbed6 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -27,8 +27,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); // compressed A auto pB = reinterpret_cast(arg->B_addr); // dense B - uint32_t N = arg->N; - uint32_t K = arg->K; + size_t N = arg->N; + size_t K = arg->K; uint32_t tid = threadIdx.x; uint32_t lane = tid % VX_CFG_NUM_THREADS; @@ -38,8 +38,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // CTA tile dimensions uint32_t cta_M = num_warps * ctx::xtileM; - uint32_t tile_row = blockIdx.y * cta_M; - uint32_t tile_col = blockIdx.x * ctx::xtileN; + size_t tile_row = blockIdx.y * cta_M; + size_t tile_col = blockIdx.x * ctx::xtileN; auto smem_base = reinterpret_cast(__local_mem()); uint32_t smem_b_off = @@ -51,28 +51,28 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); - uint32_t a_sp_stride = K / 2; + size_t a_sp_stride = K / 2; uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; - for (uint32_t k = 0; k < K; k += ctx::tileK) { + for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; // Each warp loads its compressed A tile into its private TBUF section. auto A_smem_w = reinterpret_cast( smem_base + warp_rank * per_warp_section); - for (uint32_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { - uint32_t r = i / (ctx::tileK / 2); - uint32_t c = i % (ctx::tileK / 2); + for (size_t i = lane; i < smem_a_elems; i += VX_CFG_NUM_THREADS) { + size_t r = i / (ctx::tileK / 2); + size_t c = i % (ctx::tileK / 2); A_smem_w[i] = pA[(tile_row + warp_rank * ctx::xtileM + r) * a_sp_stride + (k / 2 + c)]; } // Cooperative load: K-major B, matching the TBUF's strided descriptor. - for (uint32_t i = tid; i < smem_b_elems; i += num_threads) { - uint32_t r = i / ctx::xtileN; - uint32_t c = i % ctx::xtileN; + for (size_t i = tid; i < smem_b_elems; i += num_threads) { + size_t r = i / ctx::xtileN; + size_t c = i % ctx::xtileN; B_smem[c * ctx::tileK + r] = pB[(k + r) * N + (tile_col + c)]; } diff --git a/tests/regression/sgemm_v1/kernel.cpp b/tests/regression/sgemm_v1/kernel.cpp index 4634bae635..001651da5f 100644 --- a/tests/regression/sgemm_v1/kernel.cpp +++ b/tests/regression/sgemm_v1/kernel.cpp @@ -5,13 +5,13 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); auto C = reinterpret_cast(arg->C_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x; int row = blockIdx.y; TYPE sum(0); - for (uint32_t e = 0; e < size; ++e) { + for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } diff --git a/tests/regression/softmax/kernel.cpp b/tests/regression/softmax/kernel.cpp index 92fec90970..4d2a0ea5ac 100644 --- a/tests/regression/softmax/kernel.cpp +++ b/tests/regression/softmax/kernel.cpp @@ -5,30 +5,30 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src0_ptr = reinterpret_cast(arg->src0_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); - uint32_t num_cols = arg->num_cols; + size_t num_cols = arg->num_cols; uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; if (row >= arg->num_rows) return; - uint32_t offset = row * num_cols; + size_t offset = row * num_cols; // 1. Find the max value TYPE max = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto curr = src0_ptr[offset + i]; if (curr > max) max = curr; } // 2. Subtract max and compute exp TYPE sum = 0.0; - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { auto computed = exp(src0_ptr[offset + i] - max); src0_ptr[offset + i] = computed; sum += computed; } // 3. Normalize - for (uint32_t i = 0; i < num_cols; i++) { + for (size_t i = 0; i < num_cols; i++) { dst_ptr[offset + i] = src0_ptr[offset + i] / sum; } } diff --git a/tests/regression/stencil3d/kernel.cpp b/tests/regression/stencil3d/kernel.cpp index bab24af116..9c0c0c7bf4 100644 --- a/tests/regression/stencil3d/kernel.cpp +++ b/tests/regression/stencil3d/kernel.cpp @@ -5,7 +5,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A = reinterpret_cast(arg->A_addr); auto B = reinterpret_cast(arg->B_addr); - auto size = arg->size; + size_t size = arg->size; int col = blockIdx.x * blockDim.x + threadIdx.x; int row = blockIdx.y * blockDim.y + threadIdx.y; diff --git a/tests/regression/vecadd/kernel.cpp b/tests/regression/vecadd/kernel.cpp index a84972e50e..3660c77e59 100644 --- a/tests/regression/vecadd/kernel.cpp +++ b/tests/regression/vecadd/kernel.cpp @@ -6,5 +6,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto src1_ptr = reinterpret_cast(arg->src1_addr); auto dst_ptr = reinterpret_cast(arg->dst_addr); uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= arg->num_points) { + return; + } dst_ptr[idx] = src0_ptr[idx] + src1_ptr[idx]; } diff --git a/tests/regression/vm_stress/kernel.cpp b/tests/regression/vm_stress/kernel.cpp index c1df67de35..dc241710a1 100644 --- a/tests/regression/vm_stress/kernel.cpp +++ b/tests/regression/vm_stress/kernel.cpp @@ -14,8 +14,8 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t bias = phys_ptr[task_id % arg->phys_words]; for (uint32_t k = 0; k < arg->pages_per_task; ++k) { - uint32_t page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; - uint32_t word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); + int page = ((task_id * arg->pages_per_task + k) * arg->stride_pages) % arg->total_pages; + int word = page * WORDS_PER_PAGE + (task_id % WORDS_PER_PAGE); dst_ptr[word] = src_ptr[word] + bias; } } diff --git a/tests/riscv/common.mk b/tests/riscv/common.mk index ef9ec7332b..b3c3a9413d 100644 --- a/tests/riscv/common.mk +++ b/tests/riscv/common.mk @@ -41,8 +41,9 @@ BENCH_LIST := median qsort rsort towers memcpy multiply dhrystone spmv # (VX_MEM_IO_EXIT_CODE, VX_MEM_IO_COUT_*) via the patched crt.S + # syscalls.c, so the stamp must invalidate when those values change. # Listing $(ROOT_DIR)/sw/VX_types.h as a prerequisite forces a rebuild -# after a memory-map shift (e.g. a COUT region size change). -$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h +# after a memory-map shift (e.g. a COUT region size change), and listing +# the patch re-applies an edited patch to a fresh clone. +$(RISCV_TESTS_STAMP): $(ROOT_DIR)/sw/VX_types.h $(VORTEX_HOME)/miscs/patches/riscv-benchmarks.patch rm -rf $(RISCV_TESTS_DIR) git clone $(RISCV_TESTS_REPO) $(RISCV_TESTS_DIR) cd $(RISCV_TESTS_DIR) && git checkout --quiet $(RISCV_TESTS_COMMIT) && git submodule update --init --recursive diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index b344a6ff1a..33554509cd 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dram_sim dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/common.mk b/tests/unittest/common.mk index 4a0139e677..07887cbafd 100644 --- a/tests/unittest/common.mk +++ b/tests/unittest/common.mk @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) -CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors +CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors -Werror CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SW_COMMON_DIR) CXXFLAGS += $(CONFIGS) diff --git a/tests/unittest/dram_sim/Makefile b/tests/unittest/dram_sim/Makefile new file mode 100644 index 0000000000..7994c7a91d --- /dev/null +++ b/tests/unittest/dram_sim/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := dram_sim + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) +RAMULATOR_DIR := $(THIRD_PARTY_DIR)/ramulator + +SRCS := $(SRC_DIR)/main.cpp $(SIM_COMMON_DIR)/dram_sim.cpp + +CXXFLAGS += -I$(SIM_COMMON_DIR) +CXXFLAGS += -isystem $(RAMULATOR_DIR)/ext/spdlog/include -isystem $(RAMULATOR_DIR)/ext/yaml-cpp/include -isystem $(RAMULATOR_DIR)/src +LDFLAGS += -Wl,-rpath,$(RAMULATOR_DIR) -L$(RAMULATOR_DIR) -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state + +include ../common.mk diff --git a/tests/unittest/dram_sim/main.cpp b/tests/unittest/dram_sim/main.cpp new file mode 100644 index 0000000000..4ec135770c --- /dev/null +++ b/tests/unittest/dram_sim/main.cpp @@ -0,0 +1,230 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +using namespace vortex; + +static uint64_t g_responses = 0; + +static bool on_response(void*) { + ++g_responses; + return true; +} + +// The a100 profile: 32 HBM2 channels at 3,038 MT/s (the card's 40 at 2,430) +// behind a 1,410 MHz core, with the fixed latency (190 cycles) and request +// queue (257) of GPGPU-Sim's SM80 configuration. +static DramSim::Config a100_config() { + DramSim::Config config; + config.type = DramSim::Type::HBM2; + config.num_channels = 32; + config.block_size = 64; + config.core_clock = 1410; + config.data_rate = 3038; + config.latency = 190; + config.queue_size = 257; + return config; +} + +// Peak bytes per core cycle: a 32 B burst every two DRAM clocks per +// pseudo-channel (two per channel), or every eight clocks per GDDR6 channel. +static double peak_bandwidth(const DramSim::Config& config) { + double bytes_per_clock = (config.type == DramSim::Type::GDDR6) ? 4.0 : 32.0; + return config.num_channels * bytes_per_clock * (config.data_rate / 2.0) / config.core_clock; +} + +// Median core cycles for one read on an idle channel; `addr_of(i)` picks the +// i-th address, and the first read (cold) is not counted. +template +static double unloaded_latency(const DramSim::Config& config, F addr_of) { + DramSim dram(config); + dram.reset(); + const int n = 33; + std::vector samples; + for (int i = 0; i < n; ++i) { + g_responses = 0; + uint64_t cycles = 0; + dram.send_request(0, addr_of(i), false, on_response, nullptr); + while (g_responses == 0) { + dram.tick(); + ++cycles; + } + if (i != 0) { + samples.push_back(cycles); + } + for (int k = 0; k < 256; ++k) { + dram.tick(); + } + } + std::sort(samples.begin(), samples.end()); + return double(samples[samples.size() / 2]); +} + +// Bytes per core cycle when every channel streams `blocks` blocks, taken in +// turn from `streams` sequential streams `stride` bytes apart; +// `peak_inflight` receives the most reads outstanding at once. +static double stream_bandwidth(const DramSim::Config& config, uint32_t blocks, uint64_t* peak_inflight = nullptr, + uint32_t streams = 1, uint64_t stride = 0) { + DramSim dram(config); + dram.reset(); + g_responses = 0; + std::vector sent(config.num_channels, 0); + uint64_t total = uint64_t(blocks) * config.num_channels; + uint64_t issued = 0; + uint64_t cycles = 0; + uint64_t peak = 0; + while (g_responses < total) { + for (uint32_t ch = 0; ch < config.num_channels; ++ch) { + if (sent[ch] < blocks && dram.can_accept(ch)) { + uint64_t addr = uint64_t(sent[ch] % streams) * stride + uint64_t(sent[ch] / streams) * config.block_size; + dram.send_request(ch, addr, false, on_response, nullptr); + ++sent[ch]; + ++issued; + } + } + peak = std::max(peak, issued - g_responses); + dram.tick(); + ++cycles; + } + if (peak_inflight) { + *peak_inflight = peak; + } + return double(total) * config.block_size / cycles; +} + +// Alternates between two rows of one bank: the rows from address bit 15 on +// fold to the same bank permutation (HBM 33 = 1|1 in 5-bit chunks, GDDR6 +// 17 = 1|1 in 4-bit chunks). +static uint64_t hbm_conflict(int i) { return uint64_t(i % 2) * 33 << 15; } +static uint64_t gddr6_conflict(int i) { return uint64_t(i % 2) * 17 << 15; } + +static int g_errors = 0; + +static void check(bool cond, const char* what, double got) { + printf("%s %s (%.1f)\n", cond ? "PASS" : "FAIL", what, got); + if (!cond) { + ++g_errors; + } +} + +int main() { + auto a100 = a100_config(); + + // Unloaded latency: open-row hits, then row conflicts (precharge + activate). + double open = unloaded_latency(a100, [](int i) { return uint64_t(i % 8) * 128; }); + double conflict = unloaded_latency(a100, hbm_conflict); + check(open > 205 && open < 225, "A100 open-row read latency ~215 core cycles", open); + check(conflict > 245 && conflict < 265, "A100 row-conflict read latency ~255 core cycles", conflict); + + // The fixed latency is added exactly once per read (measured at a 1:1 clock + // ratio, so the DRAM clock phase cannot shift the comparison). + auto unit_clock = a100; + unit_clock.core_clock = unit_clock.data_rate / 2; + auto no_latency = unit_clock; + no_latency.latency = 0; + auto open_row = [](int i) { return uint64_t(i % 8) * 128; }; + double added = unloaded_latency(unit_clock, open_row) - unloaded_latency(no_latency, open_row); + check(added == 190, "fixed latency adds 190 cycles", added); + + // Every channel is an independent controller: 32 channels stream ~32x one, + // and the count need not be a power of two. + auto one = a100; + one.num_channels = 1; + double bw1 = stream_bandwidth(one, 4096); + double bw32 = stream_bandwidth(a100, 4096); + check(bw32 / bw1 > 29 && bw32 / bw1 < 33, "bandwidth scales with the channel count", bw32 / bw1); + auto five = a100; + five.num_channels = 5; + double bw5 = stream_bandwidth(five, 4096); + check(bw5 / bw1 > 4.5 && bw5 / bw1 < 5.1, "5 channels stream ~5x one", bw5 / bw1); + + // Saturated stream efficiency against the computed peak (1,103 B/cycle, the + // card's 1,555 GB/s). + double a100_peak = peak_bandwidth(a100); + check(bw32 > 0.75 * a100_peak && bw32 <= a100_peak, "A100 stream reaches 75-100% of peak", bw32); + + // The controller queue bounds the requests in flight. + auto narrow = a100; + narrow.num_channels = 1; + narrow.queue_size = 8; + uint64_t peak = 0; + stream_bandwidth(narrow, 1024, &peak); + check(peak <= 8, "queue of 8 bounds requests in flight", peak); + + // A block of two bursts returns one response. + auto wide = a100; + wide.num_channels = 1; + wide.block_size = 128; + { + DramSim dram(wide); + dram.reset(); + g_responses = 0; + dram.send_request(0, 0, false, on_response, nullptr); + for (int k = 0; k < 2000; ++k) { + dram.tick(); + } + check(g_responses == 1, "a 128 B block returns one response", double(g_responses)); + } + + // The platform default: 2 HBM2 channels clocked with a 400 MHz core. + DramSim::Config base{DramSim::Type::HBM2, 2, 64, 400, 800, 0, 0}; + double base_open = unloaded_latency(base, [](int i) { return uint64_t(i % 8) * 64; }); + double base_conflict = unloaded_latency(base, hbm_conflict); + check(base_open == 14, "default open-row read latency 14 cycles", base_open); + check(base_conflict == 26, "default row-conflict read latency 26 cycles", base_conflict); + + // Streams a power of two apart open their rows in different banks: four + // streams 1 MB apart on one channel keep over half a single stream's rate + // (an unpermuted map keeps a third). They still share banks over time, so + // each row visit pays an activation. + auto base_one = base; + base_one.num_channels = 1; + double seq = stream_bandwidth(base_one, 4096); + double strided = stream_bandwidth(base_one, 4096, nullptr, 4, 1 << 20); + check(strided > 0.55 * seq, "four streams 1 MB apart keep over half one stream's rate", strided / seq); + double bw2 = stream_bandwidth(base, 4096); + double bw1h = stream_bandwidth(base_one, 4096); + check(bw2 / bw1h > 1.8 && bw2 / bw1h < 2.05, "HBM2 2 channels stream ~2x one", bw2 / bw1h); + + // HBM3 at the h100 profile's shape: 32 channels at 6,545 MT/s behind a + // 1,980 MHz core, the card's 3.35 TB/s. + DramSim::Config h100{DramSim::Type::HBM3, 32, 64, 1980, 6545, 0, 0}; + double h100_peak = peak_bandwidth(h100); + double hbw = stream_bandwidth(h100, 4096); + check(hbw > 0.75 * h100_peak && hbw <= h100_peak, "H100 stream reaches 75-100% of peak", hbw); + + // GDDR6 at the rtx5080 profile's shape: 32 x16 channels at 15 GT/s behind a + // 2,295 MHz core: 418 B/cycle peak, the card's 960 GB/s. + DramSim::Config gddr6{DramSim::Type::GDDR6, 32, 64, 2295, 15000, 0, 0}; + double gpeak = peak_bandwidth(gddr6); + double gbw32 = stream_bandwidth(gddr6, 4096); + check(gbw32 > 0.7 * gpeak && gbw32 <= gpeak, "GDDR6 stream reaches 70-100% of peak", gbw32); + auto gddr6_one = gddr6; + gddr6_one.num_channels = 1; + double gbw1 = stream_bandwidth(gddr6_one, 4096); + check(gbw32 / gbw1 > 29 && gbw32 / gbw1 < 33, "GDDR6 bandwidth scales with the channel count", gbw32 / gbw1); + double gopen = unloaded_latency(gddr6_one, [](int i) { return uint64_t(i % 8) * 32; }); + double gconflict = unloaded_latency(gddr6_one, gddr6_conflict); + check(gopen < gconflict, "GDDR6 row conflicts cost more than row hits", gconflict - gopen); + + if (g_errors != 0) { + printf("FAILED: %d check(s)\n", g_errors); + return 1; + } + printf("PASSED!\n"); + return 0; +} diff --git a/tests/unittest/linked_list/Makefile b/tests/unittest/linked_list/Makefile new file mode 100644 index 0000000000..293746789e --- /dev/null +++ b/tests/unittest/linked_list/Makefile @@ -0,0 +1,10 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := linked_list + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +include ../common.mk diff --git a/tests/unittest/linked_list/main.cpp b/tests/unittest/linked_list/main.cpp new file mode 100644 index 0000000000..9434ed22bd --- /dev/null +++ b/tests/unittest/linked_list/main.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +using namespace vortex; + +struct Node { + int value; + LinkedListNode hook; +}; + +using List = LinkedList; + +static bool expect(const List& list, const std::vector& want, const char* what) { + std::vector got; + for (auto& n : list) { + got.push_back(n.value); + } + if (got == want) + return true; + printf("Error: %s: got {", what); + for (size_t i = 0; i < got.size(); ++i) printf("%s%d", i ? ", " : "", got[i]); + printf("}, expected {"); + for (size_t i = 0; i < want.size(); ++i) printf("%s%d", i ? ", " : "", want[i]); + printf("}\n"); + return false; +} + +int main() { + Node n[6] = {{1, {}}, {2, {}}, {3, {}}, {4, {}}, {5, {}}, {6, {}}}; + List list; + bool ok = true; + + // The simulator's timing wheel delivers same-cycle events in push_back order. + list.push_back(&n[0]); + list.push_back(&n[1]); + list.push_back(&n[2]); + ok &= expect(list, {1, 2, 3}, "push_back"); + + list.push_front(&n[3]); + ok &= expect(list, {4, 1, 2, 3}, "push_front"); + + list.insert(++list.begin(), &n[4]); + ok &= expect(list, {4, 5, 1, 2, 3}, "insert before position"); + + list.erase(list.begin()); + list.pop_back(); + ok &= expect(list, {5, 1, 2}, "erase/pop_back"); + + list.push_back(&n[5]); + ok &= expect(list, {5, 1, 2, 6}, "push_back after erase"); + + printf("%s\n", ok ? "PASSED!" : "FAILED!"); + return ok ? 0 : 1; +} diff --git a/tests/vulkan/common.mk b/tests/vulkan/common.mk index a3a6453258..ccdaeabd90 100644 --- a/tests/vulkan/common.mk +++ b/tests/vulkan/common.mk @@ -70,7 +70,7 @@ CONFIGS += -DVX_CFG_EXT_A_ENABLE GLSLC ?= glslc CC ?= cc -CFLAGS += -O2 -Wall +CFLAGS += -O2 -Wall -Wextra -Wfatal-errors -Werror LDLIBS += -lvulkan # One SPIR-V module per shader source, stage kept in the name so a diff --git a/tests/vulkan/subgroup_vs/main.c b/tests/vulkan/subgroup_vs/main.c index ae42a7f810..1538dafc3d 100644 --- a/tests/vulkan/subgroup_vs/main.c +++ b/tests/vulkan/subgroup_vs/main.c @@ -443,7 +443,7 @@ main(int argc, char **argv) return 1; } - struct sg_report r = { 0, 0, 0, 0 }; + struct sg_report r = { 0 }; int rc = render_subgroup(dev, queue, qf, &mp, vs, fs, &r); vkDestroyShaderModule(dev, vs, NULL); diff --git a/third_party/Makefile b/third_party/Makefile index a5df42e081..a32ff0ff30 100644 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -7,7 +7,21 @@ softfloat: ramulator/libramulator.so: cd ramulator && mkdir -p build && cd build && cmake .. && make -j4 -ramulator: ramulator/libramulator.so + +# The GDDR6 device model Ramulator ships but does not build. It registers with +# Ramulator's factory when the library loads, built as Ramulator builds itself. +RAMULATOR_EXT_DIR := ramulator_ext +RAMULATOR_EXT_INCS := -isystem ramulator/src -isystem ramulator/ext/spdlog/include -isystem ramulator/ext/yaml-cpp/include +RAMULATOR_EXT_CXXFLAGS := -std=c++20 -O3 -DNDEBUG -fPIC $(RAMULATOR_EXT_INCS) + +$(RAMULATOR_EXT_DIR)/gddr6.o: ramulator/src/dram/impl/GDDR6.cpp + mkdir -p $(RAMULATOR_EXT_DIR) + $(CXX) $(RAMULATOR_EXT_CXXFLAGS) -c $< -o $@ + +$(RAMULATOR_EXT_DIR)/libramulator-vortex.so: $(RAMULATOR_EXT_DIR)/gddr6.o ramulator/libramulator.so + $(CXX) -shared $(filter %.o,$^) -Lramulator -lramulator -o $@ + +ramulator: ramulator/libramulator.so $(RAMULATOR_EXT_DIR)/libramulator-vortex.so cocogfx/libcocogfx.a: $(MAKE) -C cocogfx @@ -16,6 +30,7 @@ cocogfx: cocogfx/libcocogfx.a clean: $(MAKE) -C softfloat/build/Linux-x86_64-GCC clean rm -rf ramulator/build ramulator/libramulator.so + rm -f $(RAMULATOR_EXT_DIR)/*.o $(RAMULATOR_EXT_DIR)/*.so $(MAKE) -C cocogfx clean .PHONY: all cvfpu softfloat ramulator cocogfx \ No newline at end of file